2024-11-11 18:13:14,692 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 18:13:14,708 main DEBUG Took 0.013735 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 18:13:14,709 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 18:13:14,709 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 18:13:14,710 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 18:13:14,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,721 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 18:13:14,739 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,742 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,743 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,744 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,745 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,746 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,747 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,749 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,750 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,751 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,752 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,753 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 18:13:14,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,754 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 18:13:14,756 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 18:13:14,758 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 18:13:14,760 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 18:13:14,761 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 18:13:14,762 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 18:13:14,763 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 18:13:14,779 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 18:13:14,783 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 18:13:14,785 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 18:13:14,785 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 18:13:14,786 main DEBUG createAppenders(={Console}) 2024-11-11 18:13:14,787 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 18:13:14,787 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 18:13:14,787 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 18:13:14,788 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 18:13:14,789 main DEBUG OutputStream closed 2024-11-11 18:13:14,789 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 18:13:14,789 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 18:13:14,790 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 18:13:14,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 18:13:14,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 18:13:14,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 18:13:14,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 18:13:14,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 18:13:14,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 18:13:14,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 18:13:14,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 18:13:14,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 18:13:14,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 18:13:14,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 18:13:14,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 18:13:14,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 18:13:14,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 18:13:14,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 18:13:14,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 18:13:14,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 18:13:14,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 18:13:14,911 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 18:13:14,911 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 18:13:14,912 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 18:13:14,913 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T18:13:15,351 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f 2024-11-11 18:13:15,355 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 18:13:15,356 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T18:13:15,369 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T18:13:15,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T18:13:15,420 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f, deleteOnExit=true 2024-11-11T18:13:15,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T18:13:15,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/test.cache.data in system properties and HBase conf 2024-11-11T18:13:15,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T18:13:15,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/hadoop.log.dir in system properties and HBase conf 2024-11-11T18:13:15,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T18:13:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T18:13:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T18:13:15,569 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T18:13:15,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T18:13:15,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T18:13:15,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T18:13:15,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T18:13:15,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T18:13:15,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T18:13:15,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T18:13:15,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T18:13:15,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T18:13:15,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T18:13:15,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/nfs.dump.dir in system properties and HBase conf 2024-11-11T18:13:15,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/java.io.tmpdir in system properties and HBase conf 2024-11-11T18:13:15,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T18:13:15,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T18:13:15,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T18:13:16,711 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T18:13:16,820 INFO [Time-limited test {}] log.Log(170): Logging initialized @3049ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T18:13:16,933 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T18:13:17,012 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T18:13:17,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T18:13:17,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T18:13:17,068 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T18:13:17,087 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T18:13:17,091 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/hadoop.log.dir/,AVAILABLE} 2024-11-11T18:13:17,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T18:13:17,331 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/java.io.tmpdir/jetty-localhost-41413-hadoop-hdfs-3_4_1-tests_jar-_-any-1420228937880823938/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T18:13:17,339 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:41413} 2024-11-11T18:13:17,339 INFO [Time-limited test {}] server.Server(415): Started @3570ms 2024-11-11T18:13:17,775 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T18:13:17,785 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T18:13:17,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T18:13:17,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T18:13:17,786 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T18:13:17,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/hadoop.log.dir/,AVAILABLE} 2024-11-11T18:13:17,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T18:13:17,924 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/java.io.tmpdir/jetty-localhost-46655-hadoop-hdfs-3_4_1-tests_jar-_-any-16162833823051766430/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T18:13:17,925 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:46655} 2024-11-11T18:13:17,925 INFO [Time-limited test {}] server.Server(415): Started @4156ms 2024-11-11T18:13:17,984 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T18:13:18,615 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/data/data2/current/BP-503899657-172.17.0.2-1731348796435/current, will proceed with Du for space computation calculation, 2024-11-11T18:13:18,615 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/data/data1/current/BP-503899657-172.17.0.2-1731348796435/current, will proceed with Du for space computation calculation, 2024-11-11T18:13:18,693 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T18:13:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xffc8fda1c95359d2 with lease ID 0x2417242f0f5f5cb8: Processing first storage report for DS-40c50b96-639c-4598-918d-ed4e3964e560 from datanode DatanodeRegistration(127.0.0.1:41191, datanodeUuid=89f9cc4a-99bd-4974-bb56-d9280a3d2ae7, infoPort=40557, infoSecurePort=0, ipcPort=40295, storageInfo=lv=-57;cid=testClusterID;nsid=1396407212;c=1731348796435) 2024-11-11T18:13:18,771 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xffc8fda1c95359d2 with lease ID 0x2417242f0f5f5cb8: from storage DS-40c50b96-639c-4598-918d-ed4e3964e560 node DatanodeRegistration(127.0.0.1:41191, datanodeUuid=89f9cc4a-99bd-4974-bb56-d9280a3d2ae7, infoPort=40557, infoSecurePort=0, ipcPort=40295, storageInfo=lv=-57;cid=testClusterID;nsid=1396407212;c=1731348796435), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-11-11T18:13:18,772 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xffc8fda1c95359d2 with lease ID 0x2417242f0f5f5cb8: Processing first storage report for DS-9f6a3c60-ad52-41a6-9bf4-0386060aa274 from datanode DatanodeRegistration(127.0.0.1:41191, datanodeUuid=89f9cc4a-99bd-4974-bb56-d9280a3d2ae7, infoPort=40557, infoSecurePort=0, ipcPort=40295, storageInfo=lv=-57;cid=testClusterID;nsid=1396407212;c=1731348796435) 2024-11-11T18:13:18,772 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xffc8fda1c95359d2 with lease ID 0x2417242f0f5f5cb8: from storage DS-9f6a3c60-ad52-41a6-9bf4-0386060aa274 node DatanodeRegistration(127.0.0.1:41191, datanodeUuid=89f9cc4a-99bd-4974-bb56-d9280a3d2ae7, infoPort=40557, infoSecurePort=0, ipcPort=40295, storageInfo=lv=-57;cid=testClusterID;nsid=1396407212;c=1731348796435), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T18:13:18,813 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f 2024-11-11T18:13:18,903 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/zookeeper_0, clientPort=59563, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T18:13:18,917 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59563 2024-11-11T18:13:18,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:18,937 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:19,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741825_1001 (size=7) 2024-11-11T18:13:19,615 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 with version=8 2024-11-11T18:13:19,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/hbase-staging 2024-11-11T18:13:19,707 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T18:13:19,970 INFO [Time-limited test {}] client.ConnectionUtils(128): master/90f34c5bee6a:0 server-side Connection retries=45 2024-11-11T18:13:19,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T18:13:19,986 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T18:13:19,993 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T18:13:19,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T18:13:19,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T18:13:20,179 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T18:13:20,253 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T18:13:20,266 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T18:13:20,271 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T18:13:20,304 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1002 (auto-detected) 2024-11-11T18:13:20,306 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T18:13:20,325 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41925 2024-11-11T18:13:20,350 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41925 connecting to ZooKeeper ensemble=127.0.0.1:59563 2024-11-11T18:13:20,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419250x0, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T18:13:20,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41925-0x101083c082c0000 connected 2024-11-11T18:13:20,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:20,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:20,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T18:13:20,450 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7, hbase.cluster.distributed=false 2024-11-11T18:13:20,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T18:13:20,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41925 2024-11-11T18:13:20,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41925 2024-11-11T18:13:20,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41925 2024-11-11T18:13:20,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41925 2024-11-11T18:13:20,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41925 2024-11-11T18:13:20,624 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/90f34c5bee6a:0 server-side Connection retries=45 2024-11-11T18:13:20,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T18:13:20,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T18:13:20,627 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T18:13:20,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T18:13:20,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T18:13:20,631 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T18:13:20,634 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T18:13:20,636 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42957 2024-11-11T18:13:20,639 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42957 connecting to ZooKeeper ensemble=127.0.0.1:59563 2024-11-11T18:13:20,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:20,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:20,660 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429570x0, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T18:13:20,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429570x0, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T18:13:20,662 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42957-0x101083c082c0001 connected 2024-11-11T18:13:20,667 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T18:13:20,678 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T18:13:20,681 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T18:13:20,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T18:13:20,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42957 2024-11-11T18:13:20,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42957 2024-11-11T18:13:20,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42957 2024-11-11T18:13:20,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42957 2024-11-11T18:13:20,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42957 2024-11-11T18:13:20,711 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;90f34c5bee6a:41925 2024-11-11T18:13:20,712 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:20,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T18:13:20,720 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T18:13:20,722 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:20,750 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T18:13:20,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:20,750 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:20,752 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T18:13:20,753 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/90f34c5bee6a,41925,1731348799760 from backup master directory 2024-11-11T18:13:20,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:20,759 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T18:13:20,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T18:13:20,760 WARN [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T18:13:20,760 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:20,763 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T18:13:20,765 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T18:13:20,835 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/hbase.id] with ID: 0a39067c-aaed-4091-9a8c-8edb05655f25 2024-11-11T18:13:20,835 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/.tmp/hbase.id 2024-11-11T18:13:20,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741826_1002 (size=42) 2024-11-11T18:13:20,849 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/.tmp/hbase.id]:[hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/hbase.id] 2024-11-11T18:13:20,907 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T18:13:20,913 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T18:13:20,938 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-11T18:13:20,942 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:20,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:20,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741827_1003 (size=196) 2024-11-11T18:13:21,383 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:13:21,386 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T18:13:21,404 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:21,410 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T18:13:21,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741828_1004 (size=1189) 2024-11-11T18:13:21,867 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store 2024-11-11T18:13:21,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741829_1005 (size=34) 2024-11-11T18:13:22,292 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T18:13:22,295 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:22,296 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T18:13:22,296 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:13:22,296 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:13:22,298 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T18:13:22,298 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:13:22,298 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:13:22,300 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731348802296Disabling compacts and flushes for region at 1731348802296Disabling writes for close at 1731348802298 (+2 ms)Writing region close event to WAL at 1731348802298Closed at 1731348802298 2024-11-11T18:13:22,302 WARN [master/90f34c5bee6a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/.initializing 2024-11-11T18:13:22,302 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/WALs/90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:22,311 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T18:13:22,325 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=90f34c5bee6a%2C41925%2C1731348799760, suffix=, logDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/WALs/90f34c5bee6a,41925,1731348799760, archiveDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/oldWALs, maxLogs=10 2024-11-11T18:13:22,349 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/WALs/90f34c5bee6a,41925,1731348799760/90f34c5bee6a%2C41925%2C1731348799760.1731348802330, exclude list is [], retry=0 2024-11-11T18:13:22,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41191,DS-40c50b96-639c-4598-918d-ed4e3964e560,DISK] 2024-11-11T18:13:22,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T18:13:22,405 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/WALs/90f34c5bee6a,41925,1731348799760/90f34c5bee6a%2C41925%2C1731348799760.1731348802330 2024-11-11T18:13:22,406 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40557:40557)] 2024-11-11T18:13:22,407 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:13:22,407 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:22,410 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,411 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T18:13:22,480 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:22,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:22,483 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T18:13:22,487 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:22,488 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:22,488 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T18:13:22,491 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:22,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:22,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T18:13:22,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:22,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:22,496 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,499 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,500 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,505 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,506 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,510 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T18:13:22,514 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T18:13:22,518 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:13:22,520 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71238792, jitterRate=0.06154072284698486}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T18:13:22,525 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731348802424Initializing all the Stores at 1731348802426 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348802427 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348802427Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348802428 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348802428Cleaning up temporary data from old regions at 1731348802506 (+78 ms)Region opened successfully at 1731348802525 (+19 ms) 2024-11-11T18:13:22,527 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T18:13:22,564 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a2e083d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=90f34c5bee6a/172.17.0.2:0 2024-11-11T18:13:22,596 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T18:13:22,608 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T18:13:22,609 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T18:13:22,612 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T18:13:22,613 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T18:13:22,618 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T18:13:22,619 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T18:13:22,646 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T18:13:22,655 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T18:13:22,658 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T18:13:22,661 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T18:13:22,663 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T18:13:22,665 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T18:13:22,667 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T18:13:22,670 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T18:13:22,672 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T18:13:22,673 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T18:13:22,675 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T18:13:22,694 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T18:13:22,697 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T18:13:22,701 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T18:13:22,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T18:13:22,701 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:22,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:22,704 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=90f34c5bee6a,41925,1731348799760, sessionid=0x101083c082c0000, setting cluster-up flag (Was=false) 2024-11-11T18:13:22,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:22,717 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:22,725 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T18:13:22,728 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:22,735 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:22,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:22,742 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T18:13:22,743 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:22,751 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T18:13:22,800 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(746): ClusterId : 0a39067c-aaed-4091-9a8c-8edb05655f25 2024-11-11T18:13:22,803 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T18:13:22,809 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T18:13:22,810 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T18:13:22,813 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T18:13:22,813 DEBUG [RS:0;90f34c5bee6a:42957 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9275d44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=90f34c5bee6a/172.17.0.2:0 2024-11-11T18:13:22,828 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;90f34c5bee6a:42957 2024-11-11T18:13:22,831 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T18:13:22,831 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T18:13:22,831 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T18:13:22,831 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T18:13:22,834 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(2659): reportForDuty to master=90f34c5bee6a,41925,1731348799760 with port=42957, startcode=1731348800579 2024-11-11T18:13:22,842 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T18:13:22,845 DEBUG [RS:0;90f34c5bee6a:42957 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T18:13:22,848 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T18:13:22,854 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 90f34c5bee6a,41925,1731348799760 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T18:13:22,861 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/90f34c5bee6a:0, corePoolSize=5, maxPoolSize=5 2024-11-11T18:13:22,861 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/90f34c5bee6a:0, corePoolSize=5, maxPoolSize=5 2024-11-11T18:13:22,862 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/90f34c5bee6a:0, corePoolSize=5, maxPoolSize=5 2024-11-11T18:13:22,862 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/90f34c5bee6a:0, corePoolSize=5, maxPoolSize=5 2024-11-11T18:13:22,862 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/90f34c5bee6a:0, corePoolSize=10, maxPoolSize=10 2024-11-11T18:13:22,862 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:22,863 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/90f34c5bee6a:0, corePoolSize=2, maxPoolSize=2 2024-11-11T18:13:22,863 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:22,865 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731348832865 2024-11-11T18:13:22,867 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T18:13:22,869 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T18:13:22,870 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T18:13:22,870 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T18:13:22,875 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T18:13:22,875 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T18:13:22,875 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T18:13:22,875 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T18:13:22,876 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:22,878 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:22,878 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T18:13:22,879 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T18:13:22,881 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T18:13:22,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35489, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T18:13:22,881 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T18:13:22,883 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T18:13:22,884 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T18:13:22,886 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/90f34c5bee6a:0:becomeActiveMaster-HFileCleaner.large.0-1731348802885,5,FailOnTimeoutGroup] 2024-11-11T18:13:22,886 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/90f34c5bee6a:0:becomeActiveMaster-HFileCleaner.small.0-1731348802886,5,FailOnTimeoutGroup] 2024-11-11T18:13:22,887 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:22,887 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T18:13:22,888 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41925 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:22,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741831_1007 (size=1321) 2024-11-11T18:13:22,888 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:22,889 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:22,890 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T18:13:22,890 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:13:22,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741832_1008 (size=32) 2024-11-11T18:13:22,914 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T18:13:22,915 WARN [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T18:13:23,016 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(2659): reportForDuty to master=90f34c5bee6a,41925,1731348799760 with port=42957, startcode=1731348800579 2024-11-11T18:13:23,018 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41925 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,021 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41925 {}] master.ServerManager(517): Registering regionserver=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,029 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:13:23,029 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39585 2024-11-11T18:13:23,030 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T18:13:23,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T18:13:23,034 DEBUG [RS:0;90f34c5bee6a:42957 {}] zookeeper.ZKUtil(111): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,034 WARN [RS:0;90f34c5bee6a:42957 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T18:13:23,035 INFO [RS:0;90f34c5bee6a:42957 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T18:13:23,035 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,037 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [90f34c5bee6a,42957,1731348800579] 2024-11-11T18:13:23,061 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T18:13:23,073 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T18:13:23,078 INFO [RS:0;90f34c5bee6a:42957 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T18:13:23,078 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,079 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T18:13:23,084 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T18:13:23,086 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,086 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,086 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,086 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,087 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,087 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,087 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/90f34c5bee6a:0, corePoolSize=2, maxPoolSize=2 2024-11-11T18:13:23,087 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,087 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,088 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,088 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,088 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,088 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/90f34c5bee6a:0, corePoolSize=1, maxPoolSize=1 2024-11-11T18:13:23,088 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/90f34c5bee6a:0, corePoolSize=3, maxPoolSize=3 2024-11-11T18:13:23,088 DEBUG [RS:0;90f34c5bee6a:42957 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0, corePoolSize=3, maxPoolSize=3 2024-11-11T18:13:23,089 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,089 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,090 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,090 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,090 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,090 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,42957,1731348800579-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T18:13:23,111 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T18:13:23,113 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,42957,1731348800579-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,113 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,113 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.Replication(171): 90f34c5bee6a,42957,1731348800579 started 2024-11-11T18:13:23,132 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,132 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1482): Serving as 90f34c5bee6a,42957,1731348800579, RpcServer on 90f34c5bee6a/172.17.0.2:42957, sessionid=0x101083c082c0001 2024-11-11T18:13:23,133 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T18:13:23,133 DEBUG [RS:0;90f34c5bee6a:42957 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,133 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '90f34c5bee6a,42957,1731348800579' 2024-11-11T18:13:23,133 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T18:13:23,134 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T18:13:23,135 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T18:13:23,135 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T18:13:23,135 DEBUG [RS:0;90f34c5bee6a:42957 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,135 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '90f34c5bee6a,42957,1731348800579' 2024-11-11T18:13:23,136 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T18:13:23,136 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T18:13:23,137 DEBUG [RS:0;90f34c5bee6a:42957 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T18:13:23,137 INFO [RS:0;90f34c5bee6a:42957 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T18:13:23,137 INFO [RS:0;90f34c5bee6a:42957 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T18:13:23,242 INFO [RS:0;90f34c5bee6a:42957 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T18:13:23,246 INFO [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=90f34c5bee6a%2C42957%2C1731348800579, suffix=, logDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579, archiveDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/oldWALs, maxLogs=32 2024-11-11T18:13:23,263 DEBUG [RS:0;90f34c5bee6a:42957 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579/90f34c5bee6a%2C42957%2C1731348800579.1731348803248, exclude list is [], retry=0 2024-11-11T18:13:23,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41191,DS-40c50b96-639c-4598-918d-ed4e3964e560,DISK] 2024-11-11T18:13:23,271 INFO [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579/90f34c5bee6a%2C42957%2C1731348800579.1731348803248 2024-11-11T18:13:23,272 DEBUG [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40557:40557)] 2024-11-11T18:13:23,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:23,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T18:13:23,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T18:13:23,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T18:13:23,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T18:13:23,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T18:13:23,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T18:13:23,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T18:13:23,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T18:13:23,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T18:13:23,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740 2024-11-11T18:13:23,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740 2024-11-11T18:13:23,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T18:13:23,325 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T18:13:23,326 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:13:23,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T18:13:23,332 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:13:23,333 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59270012, jitterRate=-0.11680799722671509}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:13:23,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731348803302Initializing all the Stores at 1731348803304 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348803304Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348803305 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348803305Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348803305Cleaning up temporary data from old regions at 1731348803325 (+20 ms)Region opened successfully at 1731348803335 (+10 ms) 2024-11-11T18:13:23,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T18:13:23,335 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T18:13:23,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T18:13:23,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T18:13:23,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T18:13:23,337 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T18:13:23,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731348803335Disabling compacts and flushes for region at 1731348803335Disabling writes for close at 1731348803336 (+1 ms)Writing region close event to WAL at 1731348803337 (+1 ms)Closed at 1731348803337 2024-11-11T18:13:23,341 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T18:13:23,341 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T18:13:23,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T18:13:23,357 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T18:13:23,360 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T18:13:23,512 DEBUG [90f34c5bee6a:41925 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T18:13:23,525 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,532 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 90f34c5bee6a,42957,1731348800579, state=OPENING 2024-11-11T18:13:23,537 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T18:13:23,540 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:23,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:13:23,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T18:13:23,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T18:13:23,543 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T18:13:23,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:13:23,721 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T18:13:23,725 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59807, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T18:13:23,735 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T18:13:23,736 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T18:13:23,736 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T18:13:23,739 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=90f34c5bee6a%2C42957%2C1731348800579.meta, suffix=.meta, logDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579, archiveDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/oldWALs, maxLogs=32 2024-11-11T18:13:23,756 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579/90f34c5bee6a%2C42957%2C1731348800579.meta.1731348803741.meta, exclude list is [], retry=0 2024-11-11T18:13:23,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41191,DS-40c50b96-639c-4598-918d-ed4e3964e560,DISK] 2024-11-11T18:13:23,762 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/WALs/90f34c5bee6a,42957,1731348800579/90f34c5bee6a%2C42957%2C1731348800579.meta.1731348803741.meta 2024-11-11T18:13:23,763 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40557:40557)] 2024-11-11T18:13:23,763 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:13:23,765 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T18:13:23,768 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T18:13:23,772 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T18:13:23,777 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T18:13:23,777 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:23,777 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T18:13:23,777 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T18:13:23,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T18:13:23,782 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T18:13:23,782 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T18:13:23,785 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T18:13:23,785 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,786 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T18:13:23,788 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T18:13:23,788 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T18:13:23,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T18:13:23,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:23,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T18:13:23,791 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T18:13:23,793 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740 2024-11-11T18:13:23,795 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740 2024-11-11T18:13:23,797 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T18:13:23,797 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T18:13:23,798 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:13:23,801 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T18:13:23,803 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73515586, jitterRate=0.09546759724617004}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:13:23,803 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T18:13:23,804 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731348803778Writing region info on filesystem at 1731348803778Initializing all the Stores at 1731348803780 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348803780Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348803780Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348803780Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731348803780Cleaning up temporary data from old regions at 1731348803798 (+18 ms)Running coprocessor post-open hooks at 1731348803803 (+5 ms)Region opened successfully at 1731348803804 (+1 ms) 2024-11-11T18:13:23,812 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731348803712 2024-11-11T18:13:23,823 DEBUG [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T18:13:23,824 INFO [RS_OPEN_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T18:13:23,825 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,828 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 90f34c5bee6a,42957,1731348800579, state=OPEN 2024-11-11T18:13:23,835 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T18:13:23,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T18:13:23,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T18:13:23,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T18:13:23,836 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:23,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T18:13:23,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=90f34c5bee6a,42957,1731348800579 in 291 msec 2024-11-11T18:13:23,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T18:13:23,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 497 msec 2024-11-11T18:13:23,851 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T18:13:23,851 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T18:13:23,873 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:23,875 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:23,896 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:23,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44219, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:23,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1450 sec 2024-11-11T18:13:23,929 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731348803929, completionTime=-1 2024-11-11T18:13:23,932 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T18:13:23,932 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T18:13:23,959 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-11T18:13:23,959 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731348863959 2024-11-11T18:13:23,960 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731348923960 2024-11-11T18:13:23,960 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-11T18:13:23,962 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,41925,1731348799760-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,962 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,41925,1731348799760-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,962 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,41925,1731348799760-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,964 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-90f34c5bee6a:41925, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,964 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,965 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:23,972 DEBUG [master/90f34c5bee6a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T18:13:24,014 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.253sec 2024-11-11T18:13:24,015 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T18:13:24,017 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T18:13:24,018 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T18:13:24,018 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T18:13:24,018 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T18:13:24,019 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,41925,1731348799760-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T18:13:24,020 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,41925,1731348799760-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T18:13:24,029 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T18:13:24,030 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T18:13:24,030 INFO [master/90f34c5bee6a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=90f34c5bee6a,41925,1731348799760-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T18:13:24,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f923e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:24,126 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T18:13:24,126 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T18:13:24,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:24,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:24,139 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792de1bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:24,148 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:24,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f7a5c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:24,151 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:24,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:24,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:24,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:24,197 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=319, ProcessCount=11, AvailableMemoryMB=8854 2024-11-11T18:13:24,205 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:13:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:24,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@74124cec 2024-11-11T18:13:24,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:13:24,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:13:24,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:13:24,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:13:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:24,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T18:13:24,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T18:13:24,238 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:24,240 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T18:13:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T18:13:24,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741835_1011 (size=1061) 2024-11-11T18:13:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T18:13:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T18:13:24,656 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:13:24,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741836_1012 (size=53) 2024-11-11T18:13:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T18:13:25,066 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:25,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2330eb7332d026bb8d820d13c904c2d4, disabling compactions & flushes 2024-11-11T18:13:25,067 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. after waiting 0 ms 2024-11-11T18:13:25,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,067 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2330eb7332d026bb8d820d13c904c2d4: Waiting for close lock at 1731348805066Disabling compacts and flushes for region at 1731348805066Disabling writes for close at 1731348805067 (+1 ms)Writing region close event to WAL at 1731348805067Closed at 1731348805067 2024-11-11T18:13:25,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T18:13:25,075 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731348805070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731348805070"}]},"ts":"1731348805070"} 2024-11-11T18:13:25,080 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T18:13:25,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T18:13:25,084 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348805081"}]},"ts":"1731348805081"} 2024-11-11T18:13:25,088 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T18:13:25,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, ASSIGN}] 2024-11-11T18:13:25,092 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, ASSIGN 2024-11-11T18:13:25,093 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, ASSIGN; state=OFFLINE, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=false 2024-11-11T18:13:25,245 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2330eb7332d026bb8d820d13c904c2d4, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:25,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, ASSIGN because future has completed 2024-11-11T18:13:25,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:13:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T18:13:25,411 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,412 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:13:25,412 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,412 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:25,412 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,412 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,414 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,418 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:25,418 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2330eb7332d026bb8d820d13c904c2d4 columnFamilyName A 2024-11-11T18:13:25,418 DEBUG [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:25,419 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.HStore(327): Store=2330eb7332d026bb8d820d13c904c2d4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:25,419 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,421 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:25,422 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2330eb7332d026bb8d820d13c904c2d4 columnFamilyName B 2024-11-11T18:13:25,422 DEBUG [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:25,423 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.HStore(327): Store=2330eb7332d026bb8d820d13c904c2d4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:25,423 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,425 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:25,425 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2330eb7332d026bb8d820d13c904c2d4 columnFamilyName C 2024-11-11T18:13:25,425 DEBUG [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:25,426 INFO [StoreOpener-2330eb7332d026bb8d820d13c904c2d4-1 {}] regionserver.HStore(327): Store=2330eb7332d026bb8d820d13c904c2d4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:25,426 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,427 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,428 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,429 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,431 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,432 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,433 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:13:25,436 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,440 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:13:25,441 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2330eb7332d026bb8d820d13c904c2d4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73195163, jitterRate=0.0906929224729538}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:13:25,442 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:25,443 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2330eb7332d026bb8d820d13c904c2d4: Running coprocessor pre-open hook at 1731348805413Writing region info on filesystem at 1731348805413Initializing all the Stores at 1731348805414 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348805414Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348805414Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348805414Cleaning up temporary data from old regions at 1731348805432 (+18 ms)Running coprocessor post-open hooks at 1731348805442 (+10 ms)Region opened successfully at 1731348805443 (+1 ms) 2024-11-11T18:13:25,445 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., pid=6, masterSystemTime=1731348805405 2024-11-11T18:13:25,449 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,450 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:25,451 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2330eb7332d026bb8d820d13c904c2d4, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:25,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:13:25,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T18:13:25,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 in 206 msec 2024-11-11T18:13:25,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T18:13:25,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, ASSIGN in 372 msec 2024-11-11T18:13:25,467 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T18:13:25,467 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348805467"}]},"ts":"1731348805467"} 2024-11-11T18:13:25,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T18:13:25,472 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T18:13:25,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2450 sec 2024-11-11T18:13:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T18:13:26,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:26,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ff456c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,404 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dbb004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24557cee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,409 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,415 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:13:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f152604 2024-11-11T18:13:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:13:26,420 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:13:26,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcd5136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6869e1d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,429 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2e42a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c04ffe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,432 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b371cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa5770f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e67cef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,439 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad2d2cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,441 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2763c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76916b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18731e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,448 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750f449b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e786f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,451 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aab7274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7746c0b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a97daa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,458 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbb68e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5526e600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2edce229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,464 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e84a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,466 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70eed1b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7163a69f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,470 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7466909a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,472 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12fe730b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265884f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:26,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:26,474 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb53084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:26,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:26,476 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7f5821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:26,482 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,482 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,482 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,482 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,483 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,483 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,484 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,486 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:26,494 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,496 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:26,497 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T18:13:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:13:26,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T18:13:26,514 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:26,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:26,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:13:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:26,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T18:13:26,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T18:13:26,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:26,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:26,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:26,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bb30798e5a54496b96fbf03694edf982 is 50, key is test_row_0/A:col10/1731348806558/Put/seqid=0 2024-11-11T18:13:26,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:26,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741837_1013 (size=12001) 2024-11-11T18:13:26,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bb30798e5a54496b96fbf03694edf982 2024-11-11T18:13:26,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348806674, max timestamp=1731348806712Num uniques 859; 2024-11-11T18:13:26,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:13:26,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348806674, max timestamp=1731348806712Num uniques 859; 2024-11-11T18:13:26,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:13:26,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348806674, max timestamp=1731348806712Num uniques 859; 2024-11-11T18:13:26,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:13:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348866787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348866792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348866796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348866797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348866800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T18:13:26,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/926896a3d5aa444cad5565807480e73a is 50, key is test_row_0/B:col10/1731348806558/Put/seqid=0 2024-11-11T18:13:26,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741838_1014 (size=12001) 2024-11-11T18:13:26,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/926896a3d5aa444cad5565807480e73a 2024-11-11T18:13:26,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T18:13:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:26,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:26,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:26,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:26,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/4986c5f32c4f4c2c93e3014db8c3de5d is 50, key is test_row_0/C:col10/1731348806558/Put/seqid=0 2024-11-11T18:13:26,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741839_1015 (size=12001) 2024-11-11T18:13:26,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/4986c5f32c4f4c2c93e3014db8c3de5d 2024-11-11T18:13:26,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348866917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348866917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348866919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bb30798e5a54496b96fbf03694edf982 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bb30798e5a54496b96fbf03694edf982 2024-11-11T18:13:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348866919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348866925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:26,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bb30798e5a54496b96fbf03694edf982, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T18:13:26,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/926896a3d5aa444cad5565807480e73a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/926896a3d5aa444cad5565807480e73a 2024-11-11T18:13:26,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/926896a3d5aa444cad5565807480e73a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T18:13:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/4986c5f32c4f4c2c93e3014db8c3de5d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/4986c5f32c4f4c2c93e3014db8c3de5d 2024-11-11T18:13:26,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/4986c5f32c4f4c2c93e3014db8c3de5d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T18:13:26,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 2330eb7332d026bb8d820d13c904c2d4 in 407ms, sequenceid=12, compaction requested=false 2024-11-11T18:13:26,982 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T18:13:26,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:27,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T18:13:27,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:27,023 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T18:13:27,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:27,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:27,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/580643fe3af44e3289a7014f71465464 is 50, key is test_row_0/A:col10/1731348806783/Put/seqid=0 2024-11-11T18:13:27,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741840_1016 (size=12001) 2024-11-11T18:13:27,061 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/580643fe3af44e3289a7014f71465464 2024-11-11T18:13:27,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ba6ee1ab4d5c479890508e4b4d389adb is 50, key is test_row_0/B:col10/1731348806783/Put/seqid=0 2024-11-11T18:13:27,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741841_1017 (size=12001) 2024-11-11T18:13:27,119 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ba6ee1ab4d5c479890508e4b4d389adb 2024-11-11T18:13:27,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T18:13:27,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/01a00bd13c924782893c4bf6921850df is 50, key is test_row_0/C:col10/1731348806783/Put/seqid=0 2024-11-11T18:13:27,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741842_1018 (size=12001) 2024-11-11T18:13:27,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348867157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348867159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348867163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348867179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348867190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348867296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348867296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348867305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348867306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348867315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348867516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348867525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348867526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348867527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348867527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,576 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/01a00bd13c924782893c4bf6921850df 2024-11-11T18:13:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/580643fe3af44e3289a7014f71465464 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/580643fe3af44e3289a7014f71465464 2024-11-11T18:13:27,609 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/580643fe3af44e3289a7014f71465464, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T18:13:27,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ba6ee1ab4d5c479890508e4b4d389adb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ba6ee1ab4d5c479890508e4b4d389adb 2024-11-11T18:13:27,630 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ba6ee1ab4d5c479890508e4b4d389adb, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T18:13:27,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/01a00bd13c924782893c4bf6921850df as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/01a00bd13c924782893c4bf6921850df 2024-11-11T18:13:27,650 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/01a00bd13c924782893c4bf6921850df, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T18:13:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T18:13:27,655 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 2330eb7332d026bb8d820d13c904c2d4 in 633ms, sequenceid=47, compaction requested=false 2024-11-11T18:13:27,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:27,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T18:13:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T18:13:27,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T18:13:27,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1430 sec 2024-11-11T18:13:27,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.1710 sec 2024-11-11T18:13:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:27,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:13:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:27,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:27,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5e8c8c1107ea4a319b693b1360d99e58 is 50, key is test_row_0/A:col10/1731348807179/Put/seqid=0 2024-11-11T18:13:27,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741843_1019 (size=14341) 2024-11-11T18:13:27,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5e8c8c1107ea4a319b693b1360d99e58 2024-11-11T18:13:27,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348867906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348867909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348867913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3bad7c60614d4658bad04ba298359405 is 50, key is test_row_0/B:col10/1731348807179/Put/seqid=0 2024-11-11T18:13:27,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348867903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348867919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:27,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741844_1020 (size=12001) 2024-11-11T18:13:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348868024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348868035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348868036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348868045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348868046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348868235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348868245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348868245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348868256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348868256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3bad7c60614d4658bad04ba298359405 2024-11-11T18:13:28,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/956a55f1cd2447bfbc8a51ced6e72dad is 50, key is test_row_0/C:col10/1731348807179/Put/seqid=0 2024-11-11T18:13:28,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741845_1021 (size=12001) 2024-11-11T18:13:28,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/956a55f1cd2447bfbc8a51ced6e72dad 2024-11-11T18:13:28,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5e8c8c1107ea4a319b693b1360d99e58 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5e8c8c1107ea4a319b693b1360d99e58 2024-11-11T18:13:28,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5e8c8c1107ea4a319b693b1360d99e58, entries=200, sequenceid=59, filesize=14.0 K 2024-11-11T18:13:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3bad7c60614d4658bad04ba298359405 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3bad7c60614d4658bad04ba298359405 2024-11-11T18:13:28,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3bad7c60614d4658bad04ba298359405, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T18:13:28,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/956a55f1cd2447bfbc8a51ced6e72dad as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/956a55f1cd2447bfbc8a51ced6e72dad 2024-11-11T18:13:28,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/956a55f1cd2447bfbc8a51ced6e72dad, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T18:13:28,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2330eb7332d026bb8d820d13c904c2d4 in 670ms, sequenceid=59, compaction requested=true 2024-11-11T18:13:28,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:28,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:28,510 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:28,510 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:28,515 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:28,515 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:28,517 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:28,517 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:28,517 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,518 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,518 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bb30798e5a54496b96fbf03694edf982, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/580643fe3af44e3289a7014f71465464, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5e8c8c1107ea4a319b693b1360d99e58] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=37.4 K 2024-11-11T18:13:28,518 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/926896a3d5aa444cad5565807480e73a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ba6ee1ab4d5c479890508e4b4d389adb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3bad7c60614d4658bad04ba298359405] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.2 K 2024-11-11T18:13:28,520 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb30798e5a54496b96fbf03694edf982, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731348806534 2024-11-11T18:13:28,520 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 926896a3d5aa444cad5565807480e73a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731348806534 2024-11-11T18:13:28,521 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ba6ee1ab4d5c479890508e4b4d389adb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731348806778 2024-11-11T18:13:28,521 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580643fe3af44e3289a7014f71465464, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731348806778 2024-11-11T18:13:28,522 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bad7c60614d4658bad04ba298359405, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348807175 2024-11-11T18:13:28,522 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8c8c1107ea4a319b693b1360d99e58, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348807158 2024-11-11T18:13:28,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348807879, max timestamp=1731348808555Num uniques 859; 2024-11-11T18:13:28,562 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:28,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-11T18:13:28,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731348807879, max timestamp=1731348808555Num uniques 859; 2024-11-11T18:13:28,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T18:13:28,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:28,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:28,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:28,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:28,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:28,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:28,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/987934b65d414460a9216da30fee7e73 is 50, key is test_row_0/A:col10/1731348807879/Put/seqid=0 2024-11-11T18:13:28,600 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#9 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:28,600 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:28,601 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/3829e36f949c4d74b924efca5028f2cd is 50, key is test_row_0/A:col10/1731348807179/Put/seqid=0 2024-11-11T18:13:28,601 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/b7c0b18b86a549758e7079ce5bfdfb24 is 50, key is test_row_0/B:col10/1731348807179/Put/seqid=0 2024-11-11T18:13:28,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741846_1022 (size=12104) 2024-11-11T18:13:28,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741847_1023 (size=16681) 2024-11-11T18:13:28,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/987934b65d414460a9216da30fee7e73 2024-11-11T18:13:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741848_1024 (size=12104) 2024-11-11T18:13:28,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/216a38db5b72413dbaf5ea5c962d39cc is 50, key is test_row_0/B:col10/1731348807879/Put/seqid=0 2024-11-11T18:13:28,666 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/b7c0b18b86a549758e7079ce5bfdfb24 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b7c0b18b86a549758e7079ce5bfdfb24 2024-11-11T18:13:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T18:13:28,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:28,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T18:13:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T18:13:28,673 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:28,675 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:28,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:28,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741849_1025 (size=12001) 2024-11-11T18:13:28,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/216a38db5b72413dbaf5ea5c962d39cc 2024-11-11T18:13:28,702 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into b7c0b18b86a549758e7079ce5bfdfb24(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:28,702 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:28,705 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348808507; duration=0sec 2024-11-11T18:13:28,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:28,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:28,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:28,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348868685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348868692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,717 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:28,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,717 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,718 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,718 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/4986c5f32c4f4c2c93e3014db8c3de5d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/01a00bd13c924782893c4bf6921850df, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/956a55f1cd2447bfbc8a51ced6e72dad] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.2 K 2024-11-11T18:13:28,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348868692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,719 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4986c5f32c4f4c2c93e3014db8c3de5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731348806534 2024-11-11T18:13:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348868697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,721 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a00bd13c924782893c4bf6921850df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731348806778 2024-11-11T18:13:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348868697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,723 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 956a55f1cd2447bfbc8a51ced6e72dad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348807175 2024-11-11T18:13:28,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1b4e8899da1c469294a36428e2c77a70 is 50, key is test_row_0/C:col10/1731348807879/Put/seqid=0 2024-11-11T18:13:28,751 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:28,752 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/9aa79b0d1560402fbc4f0baf609a2bdd is 50, key is test_row_0/C:col10/1731348807179/Put/seqid=0 2024-11-11T18:13:28,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741850_1026 (size=12001) 2024-11-11T18:13:28,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1b4e8899da1c469294a36428e2c77a70 2024-11-11T18:13:28,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/987934b65d414460a9216da30fee7e73 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/987934b65d414460a9216da30fee7e73 2024-11-11T18:13:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T18:13:28,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/987934b65d414460a9216da30fee7e73, entries=250, sequenceid=82, filesize=16.3 K 2024-11-11T18:13:28,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741851_1027 (size=12104) 2024-11-11T18:13:28,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/216a38db5b72413dbaf5ea5c962d39cc as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/216a38db5b72413dbaf5ea5c962d39cc 2024-11-11T18:13:28,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/216a38db5b72413dbaf5ea5c962d39cc, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T18:13:28,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1b4e8899da1c469294a36428e2c77a70 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1b4e8899da1c469294a36428e2c77a70 2024-11-11T18:13:28,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1b4e8899da1c469294a36428e2c77a70, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T18:13:28,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~203.05 KB/207928, currentSize=107.34 KB/109920 for 2330eb7332d026bb8d820d13c904c2d4 in 253ms, sequenceid=82, compaction requested=false 2024-11-11T18:13:28,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:28,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:13:28,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:28,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:28,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:28,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:28,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:28,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:28,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T18:13:28,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:28,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:28,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5130019453854474b5acc4f9dcf78c2c is 50, key is test_row_0/A:col10/1731348808676/Put/seqid=0 2024-11-11T18:13:28,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348868853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348868855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348868862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348868865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741852_1028 (size=12001) 2024-11-11T18:13:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348868869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5130019453854474b5acc4f9dcf78c2c 2024-11-11T18:13:28,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/10c4ff83f6264e3988fcb285c5185401 is 50, key is test_row_0/B:col10/1731348808676/Put/seqid=0 2024-11-11T18:13:28,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741853_1029 (size=12001) 2024-11-11T18:13:28,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/10c4ff83f6264e3988fcb285c5185401 2024-11-11T18:13:28,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/75de7339b5894245bbc51e24aeeec814 is 50, key is test_row_0/C:col10/1731348808676/Put/seqid=0 2024-11-11T18:13:28,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741854_1030 (size=12001) 2024-11-11T18:13:28,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/75de7339b5894245bbc51e24aeeec814 2024-11-11T18:13:28,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5130019453854474b5acc4f9dcf78c2c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5130019453854474b5acc4f9dcf78c2c 2024-11-11T18:13:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T18:13:28,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5130019453854474b5acc4f9dcf78c2c, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T18:13:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T18:13:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348868975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348868975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348868976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348868976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348868985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:28,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/10c4ff83f6264e3988fcb285c5185401 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/10c4ff83f6264e3988fcb285c5185401 2024-11-11T18:13:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:29,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/10c4ff83f6264e3988fcb285c5185401, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T18:13:29,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/75de7339b5894245bbc51e24aeeec814 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/75de7339b5894245bbc51e24aeeec814 2024-11-11T18:13:29,037 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/3829e36f949c4d74b924efca5028f2cd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/3829e36f949c4d74b924efca5028f2cd 2024-11-11T18:13:29,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/75de7339b5894245bbc51e24aeeec814, entries=150, sequenceid=103, filesize=11.7 K 2024-11-11T18:13:29,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 2330eb7332d026bb8d820d13c904c2d4 in 212ms, sequenceid=103, compaction requested=true 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T18:13:29,054 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 3829e36f949c4d74b924efca5028f2cd(size=11.8 K), total size for store is 39.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,054 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,054 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348808502; duration=0sec 2024-11-11T18:13:29,054 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T18:13:29,057 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:29,057 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:29,057 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-11T18:13:29,063 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T18:13:29,063 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T18:13:29,064 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. because compaction request was cancelled 2024-11-11T18:13:29,064 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:29,064 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:29,069 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:29,069 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:29,069 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,070 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/3829e36f949c4d74b924efca5028f2cd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/987934b65d414460a9216da30fee7e73, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5130019453854474b5acc4f9dcf78c2c] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=39.8 K 2024-11-11T18:13:29,072 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3829e36f949c4d74b924efca5028f2cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348807175 2024-11-11T18:13:29,073 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 987934b65d414460a9216da30fee7e73, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731348807879 2024-11-11T18:13:29,073 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5130019453854474b5acc4f9dcf78c2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731348808676 2024-11-11T18:13:29,108 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:29,109 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4549acdfb9da4093a8c14f2628d45869 is 50, key is test_row_0/A:col10/1731348808676/Put/seqid=0 2024-11-11T18:13:29,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741855_1031 (size=12207) 2024-11-11T18:13:29,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T18:13:29,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,151 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T18:13:29,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:29,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:29,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/f3689a2f879a406b87826440d95262b2 is 50, key is test_row_0/A:col10/1731348808849/Put/seqid=0 2024-11-11T18:13:29,173 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4549acdfb9da4093a8c14f2628d45869 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4549acdfb9da4093a8c14f2628d45869 2024-11-11T18:13:29,200 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 4549acdfb9da4093a8c14f2628d45869(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,200 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,200 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348809043; duration=0sec 2024-11-11T18:13:29,201 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:29,201 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:29,201 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:29,206 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/9aa79b0d1560402fbc4f0baf609a2bdd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9aa79b0d1560402fbc4f0baf609a2bdd 2024-11-11T18:13:29,207 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:29,207 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:29,207 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,219 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b7c0b18b86a549758e7079ce5bfdfb24, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/216a38db5b72413dbaf5ea5c962d39cc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/10c4ff83f6264e3988fcb285c5185401] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.3 K 2024-11-11T18:13:29,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741856_1032 (size=12001) 2024-11-11T18:13:29,225 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c0b18b86a549758e7079ce5bfdfb24, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348807175 2024-11-11T18:13:29,226 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/f3689a2f879a406b87826440d95262b2 2024-11-11T18:13:29,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:29,229 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216a38db5b72413dbaf5ea5c962d39cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731348807879 2024-11-11T18:13:29,231 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c4ff83f6264e3988fcb285c5185401, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731348808676 2024-11-11T18:13:29,244 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 9aa79b0d1560402fbc4f0baf609a2bdd(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,245 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,245 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=13, startTime=1731348808507; duration=0sec 2024-11-11T18:13:29,245 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:29,245 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:29,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f2fce18869de4739afc01fef0d89da5f is 50, key is test_row_0/B:col10/1731348808849/Put/seqid=0 2024-11-11T18:13:29,263 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:29,264 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/b467787931df40539dabb1be35f42306 is 50, key is test_row_0/B:col10/1731348808676/Put/seqid=0 2024-11-11T18:13:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741857_1033 (size=12001) 2024-11-11T18:13:29,272 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f2fce18869de4739afc01fef0d89da5f 2024-11-11T18:13:29,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:13:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T18:13:29,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741858_1034 (size=12207) 2024-11-11T18:13:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348869278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348869285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/597b11e53c1e4e97971e999f090c49dc is 50, key is test_row_0/C:col10/1731348808849/Put/seqid=0 2024-11-11T18:13:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348869295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348869297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348869298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,325 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/b467787931df40539dabb1be35f42306 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b467787931df40539dabb1be35f42306 2024-11-11T18:13:29,336 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into b467787931df40539dabb1be35f42306(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,337 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,337 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348809043; duration=0sec 2024-11-11T18:13:29,337 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:29,337 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:29,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741859_1035 (size=12001) 2024-11-11T18:13:29,348 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/597b11e53c1e4e97971e999f090c49dc 2024-11-11T18:13:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/f3689a2f879a406b87826440d95262b2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f3689a2f879a406b87826440d95262b2 2024-11-11T18:13:29,371 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f3689a2f879a406b87826440d95262b2, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T18:13:29,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f2fce18869de4739afc01fef0d89da5f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f2fce18869de4739afc01fef0d89da5f 2024-11-11T18:13:29,382 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T18:13:29,383 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f2fce18869de4739afc01fef0d89da5f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T18:13:29,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/597b11e53c1e4e97971e999f090c49dc as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/597b11e53c1e4e97971e999f090c49dc 2024-11-11T18:13:29,394 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/597b11e53c1e4e97971e999f090c49dc, entries=150, sequenceid=117, filesize=11.7 K 2024-11-11T18:13:29,397 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2330eb7332d026bb8d820d13c904c2d4 in 245ms, sequenceid=117, compaction requested=true 2024-11-11T18:13:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T18:13:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T18:13:29,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T18:13:29,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 725 msec 2024-11-11T18:13:29,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 740 msec 2024-11-11T18:13:29,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348809277, max timestamp=1731348809411Num uniques 859; 2024-11-11T18:13:29,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:29,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-11T18:13:29,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348809277, max timestamp=1731348809413Num uniques 859; 2024-11-11T18:13:29,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:29,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:29,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:29,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/dc897c7b5f104c3fb63f34d0b3c5680c is 50, key is test_row_0/A:col10/1731348809413/Put/seqid=0 2024-11-11T18:13:29,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741860_1036 (size=12151) 2024-11-11T18:13:29,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/dc897c7b5f104c3fb63f34d0b3c5680c 2024-11-11T18:13:29,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/97c2ec3977f04f69a193130505c232bb is 50, key is test_row_0/B:col10/1731348809413/Put/seqid=0 2024-11-11T18:13:29,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741861_1037 (size=12151) 2024-11-11T18:13:29,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/97c2ec3977f04f69a193130505c232bb 2024-11-11T18:13:29,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/92675d5d4e8f442dbd371b79fe3b6189 is 50, key is test_row_0/C:col10/1731348809413/Put/seqid=0 2024-11-11T18:13:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741862_1038 (size=12151) 2024-11-11T18:13:29,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/92675d5d4e8f442dbd371b79fe3b6189 2024-11-11T18:13:29,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/dc897c7b5f104c3fb63f34d0b3c5680c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/dc897c7b5f104c3fb63f34d0b3c5680c 2024-11-11T18:13:29,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/dc897c7b5f104c3fb63f34d0b3c5680c, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T18:13:29,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/97c2ec3977f04f69a193130505c232bb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/97c2ec3977f04f69a193130505c232bb 2024-11-11T18:13:29,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/97c2ec3977f04f69a193130505c232bb, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T18:13:29,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/92675d5d4e8f442dbd371b79fe3b6189 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/92675d5d4e8f442dbd371b79fe3b6189 2024-11-11T18:13:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348869558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/92675d5d4e8f442dbd371b79fe3b6189, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T18:13:29,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 2330eb7332d026bb8d820d13c904c2d4 in 161ms, sequenceid=142, compaction requested=true 2024-11-11T18:13:29,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:29,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:29,583 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:29,583 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:29,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:29,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:29,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:29,586 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:29,586 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:29,587 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,587 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4549acdfb9da4093a8c14f2628d45869, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f3689a2f879a406b87826440d95262b2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/dc897c7b5f104c3fb63f34d0b3c5680c] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.5 K 2024-11-11T18:13:29,587 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:29,587 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:29,587 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,587 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b467787931df40539dabb1be35f42306, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f2fce18869de4739afc01fef0d89da5f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/97c2ec3977f04f69a193130505c232bb] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.5 K 2024-11-11T18:13:29,588 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4549acdfb9da4093a8c14f2628d45869, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731348808676 2024-11-11T18:13:29,589 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3689a2f879a406b87826440d95262b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731348808849 2024-11-11T18:13:29,590 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b467787931df40539dabb1be35f42306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731348808676 2024-11-11T18:13:29,591 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc897c7b5f104c3fb63f34d0b3c5680c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731348809289 2024-11-11T18:13:29,593 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f2fce18869de4739afc01fef0d89da5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731348808849 2024-11-11T18:13:29,594 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 97c2ec3977f04f69a193130505c232bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731348809289 2024-11-11T18:13:29,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348809559, max timestamp=1731348809591Num uniques 859; 2024-11-11T18:13:29,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:29,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348809559, max timestamp=1731348809591Num uniques 859; 2024-11-11T18:13:29,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:29,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=71.56 KB heapSize=204.02 KB 2024-11-11T18:13:29,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:29,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:29,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:29,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:29,619 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#26 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:29,620 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c810a7382bea4eb8a503eebba77dd727 is 50, key is test_row_0/A:col10/1731348809413/Put/seqid=0 2024-11-11T18:13:29,624 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:29,625 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/e5884eb9d4114752a7f54d1453ab785c is 50, key is test_row_0/B:col10/1731348809413/Put/seqid=0 2024-11-11T18:13:29,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/6e8787d172ee42089aca40653adba2bb is 50, key is test_row_0/A:col10/1731348809585/Put/seqid=0 2024-11-11T18:13:29,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741864_1040 (size=12459) 2024-11-11T18:13:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741863_1039 (size=12459) 2024-11-11T18:13:29,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348869665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,682 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/e5884eb9d4114752a7f54d1453ab785c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/e5884eb9d4114752a7f54d1453ab785c 2024-11-11T18:13:29,689 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c810a7382bea4eb8a503eebba77dd727 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c810a7382bea4eb8a503eebba77dd727 2024-11-11T18:13:29,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348869672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348869678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348869681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741865_1041 (size=12151) 2024-11-11T18:13:29,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/6e8787d172ee42089aca40653adba2bb 2024-11-11T18:13:29,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348869697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,706 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into e5884eb9d4114752a7f54d1453ab785c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,706 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348809583; duration=0sec 2024-11-11T18:13:29,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:29,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:29,706 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T18:13:29,710 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into c810a7382bea4eb8a503eebba77dd727(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,710 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,710 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348809583; duration=0sec 2024-11-11T18:13:29,710 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:29,711 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:29,711 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60258 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T18:13:29,711 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:29,711 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,712 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9aa79b0d1560402fbc4f0baf609a2bdd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1b4e8899da1c469294a36428e2c77a70, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/75de7339b5894245bbc51e24aeeec814, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/597b11e53c1e4e97971e999f090c49dc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/92675d5d4e8f442dbd371b79fe3b6189] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=58.8 K 2024-11-11T18:13:29,715 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa79b0d1560402fbc4f0baf609a2bdd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348807175 2024-11-11T18:13:29,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a1d81ed5ebde4bf6a6046d955965891d is 50, key is test_row_0/B:col10/1731348809585/Put/seqid=0 2024-11-11T18:13:29,720 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4e8899da1c469294a36428e2c77a70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731348807879 2024-11-11T18:13:29,720 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 75de7339b5894245bbc51e24aeeec814, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731348808676 2024-11-11T18:13:29,721 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 597b11e53c1e4e97971e999f090c49dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731348808849 2024-11-11T18:13:29,722 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 92675d5d4e8f442dbd371b79fe3b6189, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731348809289 2024-11-11T18:13:29,765 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:29,769 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/6a60fea6025e4a28989141935e3e9ab9 is 50, key is test_row_0/C:col10/1731348809413/Put/seqid=0 2024-11-11T18:13:29,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741866_1042 (size=12151) 2024-11-11T18:13:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348869785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741867_1043 (size=12425) 2024-11-11T18:13:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T18:13:29,805 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:29,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T18:13:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:29,810 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:29,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:29,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:29,817 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/6a60fea6025e4a28989141935e3e9ab9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/6a60fea6025e4a28989141935e3e9ab9 2024-11-11T18:13:29,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348869805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348869805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348869806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,833 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 6a60fea6025e4a28989141935e3e9ab9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:29,833 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:29,833 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=11, startTime=1731348809584; duration=0sec 2024-11-11T18:13:29,834 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:29,834 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:29,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348869914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:29,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:29,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:29,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:29,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348869995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348870035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348870035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348870036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a1d81ed5ebde4bf6a6046d955965891d 2024-11-11T18:13:30,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/0ccec2a2cd7f4c289dc0d39d628d1368 is 50, key is test_row_0/C:col10/1731348809585/Put/seqid=0 2024-11-11T18:13:30,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741868_1044 (size=12151) 2024-11-11T18:13:30,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348870226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,250 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T18:13:30,250 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T18:13:30,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T18:13:30,253 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T18:13:30,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T18:13:30,254 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T18:13:30,255 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T18:13:30,255 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T18:13:30,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:30,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:30,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348870305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348870357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348870358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348870358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:30,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:30,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:30,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:30,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/0ccec2a2cd7f4c289dc0d39d628d1368 2024-11-11T18:13:30,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/6e8787d172ee42089aca40653adba2bb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6e8787d172ee42089aca40653adba2bb 2024-11-11T18:13:30,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6e8787d172ee42089aca40653adba2bb, entries=150, sequenceid=165, filesize=11.9 K 2024-11-11T18:13:30,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a1d81ed5ebde4bf6a6046d955965891d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1d81ed5ebde4bf6a6046d955965891d 2024-11-11T18:13:30,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1d81ed5ebde4bf6a6046d955965891d, entries=150, sequenceid=165, filesize=11.9 K 2024-11-11T18:13:30,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/0ccec2a2cd7f4c289dc0d39d628d1368 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/0ccec2a2cd7f4c289dc0d39d628d1368 2024-11-11T18:13:30,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/0ccec2a2cd7f4c289dc0d39d628d1368, entries=150, sequenceid=165, filesize=11.9 K 2024-11-11T18:13:30,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for 2330eb7332d026bb8d820d13c904c2d4 in 1078ms, sequenceid=165, compaction requested=false 2024-11-11T18:13:30,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:30,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:13:30,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:30,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:30,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:30,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:30,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:30,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:30,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a5600f058f6d4647b45d87c5ba6cab36 is 50, key is test_row_0/A:col10/1731348809669/Put/seqid=0 2024-11-11T18:13:30,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:30,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:30,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741869_1045 (size=14541) 2024-11-11T18:13:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348870790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348870835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348870865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348870874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348870874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:30,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:31,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:31,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:31,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348871115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a5600f058f6d4647b45d87c5ba6cab36 2024-11-11T18:13:31,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/948db61347174b4c9ed1556662944d70 is 50, key is test_row_0/B:col10/1731348809669/Put/seqid=0 2024-11-11T18:13:31,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741870_1046 (size=12151) 2024-11-11T18:13:31,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/948db61347174b4c9ed1556662944d70 2024-11-11T18:13:31,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:31,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/2cfdb5bc766c4d4fa5371deff2fbb210 is 50, key is test_row_0/C:col10/1731348809669/Put/seqid=0 2024-11-11T18:13:31,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741871_1047 (size=12151) 2024-11-11T18:13:31,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/2cfdb5bc766c4d4fa5371deff2fbb210 2024-11-11T18:13:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a5600f058f6d4647b45d87c5ba6cab36 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a5600f058f6d4647b45d87c5ba6cab36 2024-11-11T18:13:31,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a5600f058f6d4647b45d87c5ba6cab36, entries=200, sequenceid=188, filesize=14.2 K 2024-11-11T18:13:31,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/948db61347174b4c9ed1556662944d70 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/948db61347174b4c9ed1556662944d70 2024-11-11T18:13:31,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/948db61347174b4c9ed1556662944d70, entries=150, sequenceid=188, filesize=11.9 K 2024-11-11T18:13:31,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/2cfdb5bc766c4d4fa5371deff2fbb210 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2cfdb5bc766c4d4fa5371deff2fbb210 2024-11-11T18:13:31,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2cfdb5bc766c4d4fa5371deff2fbb210, entries=150, sequenceid=188, filesize=11.9 K 2024-11-11T18:13:31,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2330eb7332d026bb8d820d13c904c2d4 in 563ms, sequenceid=188, compaction requested=true 2024-11-11T18:13:31,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:31,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:31,299 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:31,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:31,300 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:31,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:31,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:31,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:31,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:31,302 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:31,302 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:31,302 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,302 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/e5884eb9d4114752a7f54d1453ab785c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1d81ed5ebde4bf6a6046d955965891d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/948db61347174b4c9ed1556662944d70] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.9 K 2024-11-11T18:13:31,304 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e5884eb9d4114752a7f54d1453ab785c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731348809289 2024-11-11T18:13:31,304 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:31,305 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:31,305 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,305 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c810a7382bea4eb8a503eebba77dd727, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6e8787d172ee42089aca40653adba2bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a5600f058f6d4647b45d87c5ba6cab36] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=38.2 K 2024-11-11T18:13:31,305 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a1d81ed5ebde4bf6a6046d955965891d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731348809585 2024-11-11T18:13:31,306 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c810a7382bea4eb8a503eebba77dd727, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731348809289 2024-11-11T18:13:31,306 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 948db61347174b4c9ed1556662944d70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731348809661 2024-11-11T18:13:31,307 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e8787d172ee42089aca40653adba2bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731348809585 2024-11-11T18:13:31,310 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5600f058f6d4647b45d87c5ba6cab36, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731348809661 2024-11-11T18:13:31,332 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:31,333 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/562945a4df644d5d98668f0f6cf88b12 is 50, key is test_row_0/A:col10/1731348809669/Put/seqid=0 2024-11-11T18:13:31,346 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:31,347 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ae2a82f0206c400d82fa15af2c3bd69a is 50, key is test_row_0/B:col10/1731348809669/Put/seqid=0 2024-11-11T18:13:31,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741872_1048 (size=12561) 2024-11-11T18:13:31,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T18:13:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,370 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:13:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:31,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:31,371 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/562945a4df644d5d98668f0f6cf88b12 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/562945a4df644d5d98668f0f6cf88b12 2024-11-11T18:13:31,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741873_1049 (size=12561) 2024-11-11T18:13:31,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bd0517d0cbb248ac8704a9896ba32a1d is 50, key is test_row_0/A:col10/1731348810784/Put/seqid=0 2024-11-11T18:13:31,381 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 562945a4df644d5d98668f0f6cf88b12(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:31,382 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:31,382 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348811299; duration=0sec 2024-11-11T18:13:31,382 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:31,382 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:31,382 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:31,384 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:31,384 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:31,385 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,385 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/6a60fea6025e4a28989141935e3e9ab9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/0ccec2a2cd7f4c289dc0d39d628d1368, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2cfdb5bc766c4d4fa5371deff2fbb210] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=35.9 K 2024-11-11T18:13:31,387 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a60fea6025e4a28989141935e3e9ab9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731348809289 2024-11-11T18:13:31,388 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ccec2a2cd7f4c289dc0d39d628d1368, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731348809585 2024-11-11T18:13:31,389 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cfdb5bc766c4d4fa5371deff2fbb210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731348809661 2024-11-11T18:13:31,390 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ae2a82f0206c400d82fa15af2c3bd69a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ae2a82f0206c400d82fa15af2c3bd69a 2024-11-11T18:13:31,401 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into ae2a82f0206c400d82fa15af2c3bd69a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:31,401 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:31,401 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348811299; duration=0sec 2024-11-11T18:13:31,401 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:31,401 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:31,406 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:31,407 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/a472dbf39e2542fca747798ba5dd2a40 is 50, key is test_row_0/C:col10/1731348809669/Put/seqid=0 2024-11-11T18:13:31,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741874_1050 (size=12151) 2024-11-11T18:13:31,415 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bd0517d0cbb248ac8704a9896ba32a1d 2024-11-11T18:13:31,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:31,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741875_1051 (size=12527) 2024-11-11T18:13:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d14f778f186742b897e2d23df5fb7a05 is 50, key is test_row_0/B:col10/1731348810784/Put/seqid=0 2024-11-11T18:13:31,453 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/a472dbf39e2542fca747798ba5dd2a40 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/a472dbf39e2542fca747798ba5dd2a40 2024-11-11T18:13:31,465 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into a472dbf39e2542fca747798ba5dd2a40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:31,466 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:31,466 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=13, startTime=1731348811300; duration=0sec 2024-11-11T18:13:31,466 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:31,466 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:31,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741876_1052 (size=12151) 2024-11-11T18:13:31,471 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d14f778f186742b897e2d23df5fb7a05 2024-11-11T18:13:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d576e129a07e40759d409c779e99f380 is 50, key is test_row_0/C:col10/1731348810784/Put/seqid=0 2024-11-11T18:13:31,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741877_1053 (size=12151) 2024-11-11T18:13:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348871546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348871655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348871854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,863 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in 2330eb7332d026bb8d820d13c904c2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:13:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348871865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348871874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,878 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in 2330eb7332d026bb8d820d13c904c2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:13:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348871894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348871895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:31,898 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 2330eb7332d026bb8d820d13c904c2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:13:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,899 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 2330eb7332d026bb8d820d13c904c2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:31,938 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d576e129a07e40759d409c779e99f380 2024-11-11T18:13:31,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bd0517d0cbb248ac8704a9896ba32a1d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bd0517d0cbb248ac8704a9896ba32a1d 2024-11-11T18:13:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:31,963 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bd0517d0cbb248ac8704a9896ba32a1d, entries=150, sequenceid=200, filesize=11.9 K 2024-11-11T18:13:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d14f778f186742b897e2d23df5fb7a05 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d14f778f186742b897e2d23df5fb7a05 2024-11-11T18:13:31,973 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d14f778f186742b897e2d23df5fb7a05, entries=150, sequenceid=200, filesize=11.9 K 2024-11-11T18:13:31,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d576e129a07e40759d409c779e99f380 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d576e129a07e40759d409c779e99f380 2024-11-11T18:13:31,983 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d576e129a07e40759d409c779e99f380, entries=150, sequenceid=200, filesize=11.9 K 2024-11-11T18:13:31,987 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2330eb7332d026bb8d820d13c904c2d4 in 617ms, sequenceid=200, compaction requested=false 2024-11-11T18:13:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T18:13:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T18:13:31,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T18:13:31,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-11T18:13:32,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1930 sec 2024-11-11T18:13:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:32,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T18:13:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:32,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/32a6d945fb974fedb7fa96c2c1b758d3 is 50, key is test_row_0/A:col10/1731348811543/Put/seqid=0 2024-11-11T18:13:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741878_1054 (size=14541) 2024-11-11T18:13:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348872240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:32,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348872355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348872564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:32,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/32a6d945fb974fedb7fa96c2c1b758d3 2024-11-11T18:13:32,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/45f0f41682cb4269aa17884226f5bbd3 is 50, key is test_row_0/B:col10/1731348811543/Put/seqid=0 2024-11-11T18:13:32,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741879_1055 (size=12151) 2024-11-11T18:13:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:32,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348872875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:32,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:33,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/45f0f41682cb4269aa17884226f5bbd3 2024-11-11T18:13:33,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d981279fe0dc42988849f2822daeec9f is 50, key is test_row_0/C:col10/1731348811543/Put/seqid=0 2024-11-11T18:13:33,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741880_1056 (size=12151) 2024-11-11T18:13:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348873394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:33,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d981279fe0dc42988849f2822daeec9f 2024-11-11T18:13:33,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/32a6d945fb974fedb7fa96c2c1b758d3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/32a6d945fb974fedb7fa96c2c1b758d3 2024-11-11T18:13:33,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/32a6d945fb974fedb7fa96c2c1b758d3, entries=200, sequenceid=224, filesize=14.2 K 2024-11-11T18:13:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/45f0f41682cb4269aa17884226f5bbd3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/45f0f41682cb4269aa17884226f5bbd3 2024-11-11T18:13:33,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/45f0f41682cb4269aa17884226f5bbd3, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T18:13:33,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d981279fe0dc42988849f2822daeec9f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d981279fe0dc42988849f2822daeec9f 2024-11-11T18:13:33,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d981279fe0dc42988849f2822daeec9f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T18:13:33,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2330eb7332d026bb8d820d13c904c2d4 in 1311ms, sequenceid=224, compaction requested=true 2024-11-11T18:13:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:33,489 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:33,489 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:33,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:33,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:33,491 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:33,491 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:33,491 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:33,491 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/562945a4df644d5d98668f0f6cf88b12, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bd0517d0cbb248ac8704a9896ba32a1d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/32a6d945fb974fedb7fa96c2c1b758d3] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=38.3 K 2024-11-11T18:13:33,491 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:33,491 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:33,491 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:33,492 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ae2a82f0206c400d82fa15af2c3bd69a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d14f778f186742b897e2d23df5fb7a05, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/45f0f41682cb4269aa17884226f5bbd3] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=36.0 K 2024-11-11T18:13:33,492 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562945a4df644d5d98668f0f6cf88b12, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731348809661 2024-11-11T18:13:33,492 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ae2a82f0206c400d82fa15af2c3bd69a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731348809661 2024-11-11T18:13:33,492 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0517d0cbb248ac8704a9896ba32a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731348810769 2024-11-11T18:13:33,493 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d14f778f186742b897e2d23df5fb7a05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731348810769 2024-11-11T18:13:33,493 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a6d945fb974fedb7fa96c2c1b758d3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731348811522 2024-11-11T18:13:33,493 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 45f0f41682cb4269aa17884226f5bbd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731348811528 2024-11-11T18:13:33,513 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:33,514 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5fefcd432bed4135acce288aa8e9d973 is 50, key is test_row_0/A:col10/1731348811543/Put/seqid=0 2024-11-11T18:13:33,518 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:33,520 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/8d73efa896b2447ca57964a003fd845c is 50, key is test_row_0/B:col10/1731348811543/Put/seqid=0 2024-11-11T18:13:33,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741881_1057 (size=12663) 2024-11-11T18:13:33,539 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5fefcd432bed4135acce288aa8e9d973 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5fefcd432bed4135acce288aa8e9d973 2024-11-11T18:13:33,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741882_1058 (size=12663) 2024-11-11T18:13:33,569 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 5fefcd432bed4135acce288aa8e9d973(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:33,569 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/8d73efa896b2447ca57964a003fd845c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8d73efa896b2447ca57964a003fd845c 2024-11-11T18:13:33,570 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:33,570 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348813489; duration=0sec 2024-11-11T18:13:33,571 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:33,571 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:33,571 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:33,574 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:33,574 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:33,574 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:33,575 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/a472dbf39e2542fca747798ba5dd2a40, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d576e129a07e40759d409c779e99f380, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d981279fe0dc42988849f2822daeec9f] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=36.0 K 2024-11-11T18:13:33,575 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a472dbf39e2542fca747798ba5dd2a40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731348809661 2024-11-11T18:13:33,576 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d576e129a07e40759d409c779e99f380, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731348810769 2024-11-11T18:13:33,576 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d981279fe0dc42988849f2822daeec9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731348811528 2024-11-11T18:13:33,583 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 8d73efa896b2447ca57964a003fd845c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:33,583 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:33,583 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348813489; duration=0sec 2024-11-11T18:13:33,583 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:33,583 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:33,600 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#46 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:33,601 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/67ff1dc8096f4c4d9258fec97bc2933d is 50, key is test_row_0/C:col10/1731348811543/Put/seqid=0 2024-11-11T18:13:33,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741883_1059 (size=12629) 2024-11-11T18:13:33,643 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/67ff1dc8096f4c4d9258fec97bc2933d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/67ff1dc8096f4c4d9258fec97bc2933d 2024-11-11T18:13:33,653 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 67ff1dc8096f4c4d9258fec97bc2933d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:33,653 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:33,653 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=13, startTime=1731348813489; duration=0sec 2024-11-11T18:13:33,653 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:33,653 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:33,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:13:33,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:33,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:33,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:33,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/7c24d7ede1dc4b7e898f5f7aca115bde is 50, key is test_row_0/A:col10/1731348813875/Put/seqid=0 2024-11-11T18:13:33,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741884_1060 (size=9757) 2024-11-11T18:13:33,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/7c24d7ede1dc4b7e898f5f7aca115bde 2024-11-11T18:13:33,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d40e4abdfc6a46719dba43e67c501946 is 50, key is test_row_0/B:col10/1731348813875/Put/seqid=0 2024-11-11T18:13:33,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741885_1061 (size=9757) 2024-11-11T18:13:33,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348873926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:33,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348873929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:33,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348873930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:33,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348873931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:33,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T18:13:33,965 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:33,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T18:13:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T18:13:33,971 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:33,972 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:33,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:34,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348874035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348874035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348874044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348874044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T18:13:34,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:34,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348874244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348874255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348874256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348874256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:34,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:34,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T18:13:34,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d40e4abdfc6a46719dba43e67c501946 2024-11-11T18:13:34,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/069bb088dc48439ab73fd29189dcb768 is 50, key is test_row_0/C:col10/1731348813875/Put/seqid=0 2024-11-11T18:13:34,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741886_1062 (size=9757) 2024-11-11T18:13:34,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/069bb088dc48439ab73fd29189dcb768 2024-11-11T18:13:34,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/7c24d7ede1dc4b7e898f5f7aca115bde as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/7c24d7ede1dc4b7e898f5f7aca115bde 2024-11-11T18:13:34,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/7c24d7ede1dc4b7e898f5f7aca115bde, entries=100, sequenceid=239, filesize=9.5 K 2024-11-11T18:13:34,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d40e4abdfc6a46719dba43e67c501946 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d40e4abdfc6a46719dba43e67c501946 2024-11-11T18:13:34,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d40e4abdfc6a46719dba43e67c501946, entries=100, sequenceid=239, filesize=9.5 K 2024-11-11T18:13:34,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/069bb088dc48439ab73fd29189dcb768 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/069bb088dc48439ab73fd29189dcb768 2024-11-11T18:13:34,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/069bb088dc48439ab73fd29189dcb768, entries=100, sequenceid=239, filesize=9.5 K 2024-11-11T18:13:34,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2330eb7332d026bb8d820d13c904c2d4 in 518ms, sequenceid=239, compaction requested=false 2024-11-11T18:13:34,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:34,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348813909, max timestamp=1731348814405Num uniques 859; 2024-11-11T18:13:34,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T18:13:34,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:34,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348813909, max timestamp=1731348814405Num uniques 859; 2024-11-11T18:13:34,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:34,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:34,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:34,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:34,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:34,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:34,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/da8cceff1b9f46df8f377d3773dc6b04 is 50, key is test_row_0/A:col10/1731348813909/Put/seqid=0 2024-11-11T18:13:34,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741887_1063 (size=12251) 2024-11-11T18:13:34,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/da8cceff1b9f46df8f377d3773dc6b04 2024-11-11T18:13:34,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:34,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:34,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/eaa0c1cdcc594738a2a9d4d74250bcc5 is 50, key is test_row_0/B:col10/1731348813909/Put/seqid=0 2024-11-11T18:13:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741888_1064 (size=12251) 2024-11-11T18:13:34,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/eaa0c1cdcc594738a2a9d4d74250bcc5 2024-11-11T18:13:34,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/79740674281546cead3fd9247fd9638f is 50, key is test_row_0/C:col10/1731348813909/Put/seqid=0 2024-11-11T18:13:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741889_1065 (size=12251) 2024-11-11T18:13:34,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/79740674281546cead3fd9247fd9638f 2024-11-11T18:13:34,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/da8cceff1b9f46df8f377d3773dc6b04 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/da8cceff1b9f46df8f377d3773dc6b04 2024-11-11T18:13:34,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/da8cceff1b9f46df8f377d3773dc6b04, entries=150, sequenceid=260, filesize=12.0 K 2024-11-11T18:13:34,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/eaa0c1cdcc594738a2a9d4d74250bcc5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/eaa0c1cdcc594738a2a9d4d74250bcc5 2024-11-11T18:13:34,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/eaa0c1cdcc594738a2a9d4d74250bcc5, entries=150, sequenceid=260, filesize=12.0 K 2024-11-11T18:13:34,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/79740674281546cead3fd9247fd9638f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/79740674281546cead3fd9247fd9638f 2024-11-11T18:13:34,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348814500, max timestamp=1731348814540Num uniques 859; 2024-11-11T18:13:34,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:34,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348814500, max timestamp=1731348814540Num uniques 859; 2024-11-11T18:13:34,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:34,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348814500, max timestamp=1731348814540Num uniques 859; 2024-11-11T18:13:34,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:34,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/79740674281546cead3fd9247fd9638f, entries=150, sequenceid=260, filesize=12.0 K 2024-11-11T18:13:34,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=26.84 KB/27480 for 2330eb7332d026bb8d820d13c904c2d4 in 141ms, sequenceid=260, compaction requested=true 2024-11-11T18:13:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:34,551 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:34,551 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:34,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:34,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:34,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:34,553 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:34,553 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:34,553 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,554 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8d73efa896b2447ca57964a003fd845c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d40e4abdfc6a46719dba43e67c501946, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/eaa0c1cdcc594738a2a9d4d74250bcc5] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=33.9 K 2024-11-11T18:13:34,554 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:34,555 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:34,555 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,555 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5fefcd432bed4135acce288aa8e9d973, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/7c24d7ede1dc4b7e898f5f7aca115bde, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/da8cceff1b9f46df8f377d3773dc6b04] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=33.9 K 2024-11-11T18:13:34,555 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d73efa896b2447ca57964a003fd845c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731348811528 2024-11-11T18:13:34,556 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fefcd432bed4135acce288aa8e9d973, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731348811528 2024-11-11T18:13:34,556 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d40e4abdfc6a46719dba43e67c501946, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731348812238 2024-11-11T18:13:34,557 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c24d7ede1dc4b7e898f5f7aca115bde, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731348812238 2024-11-11T18:13:34,557 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa0c1cdcc594738a2a9d4d74250bcc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731348813909 2024-11-11T18:13:34,558 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting da8cceff1b9f46df8f377d3773dc6b04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731348813909 2024-11-11T18:13:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:34,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-11T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:34,579 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:34,580 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/91c620d174394d23afda3a08f959b999 is 50, key is test_row_0/B:col10/1731348813909/Put/seqid=0 2024-11-11T18:13:34,585 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:34,585 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b9193c478b6b4517bbc382e916633b3b is 50, key is test_row_0/A:col10/1731348813909/Put/seqid=0 2024-11-11T18:13:34,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6 is 50, key is test_row_0/A:col10/1731348814559/Put/seqid=0 2024-11-11T18:13:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T18:13:34,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:34,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:34,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731348814582, max timestamp=1731348814610Num uniques 859; 2024-11-11T18:13:34,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-11T18:13:34,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731348814582, max timestamp=1731348814610Num uniques 859; 2024-11-11T18:13:34,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-11T18:13:34,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731348814582, max timestamp=1731348814610Num uniques 859; 2024-11-11T18:13:34,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-11T18:13:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741890_1066 (size=12865) 2024-11-11T18:13:34,638 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/91c620d174394d23afda3a08f959b999 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/91c620d174394d23afda3a08f959b999 2024-11-11T18:13:34,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348874635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,648 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 91c620d174394d23afda3a08f959b999(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:34,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:34,648 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348814551; duration=0sec 2024-11-11T18:13:34,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:34,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:34,649 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:34,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741891_1067 (size=12865) 2024-11-11T18:13:34,651 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:34,651 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:34,651 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,652 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/67ff1dc8096f4c4d9258fec97bc2933d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/069bb088dc48439ab73fd29189dcb768, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/79740674281546cead3fd9247fd9638f] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=33.8 K 2024-11-11T18:13:34,653 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ff1dc8096f4c4d9258fec97bc2933d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731348811528 2024-11-11T18:13:34,654 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 069bb088dc48439ab73fd29189dcb768, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731348812238 2024-11-11T18:13:34,654 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 79740674281546cead3fd9247fd9638f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731348813909 2024-11-11T18:13:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348874642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348874638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348874645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348874648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741892_1068 (size=14741) 2024-11-11T18:13:34,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6 2024-11-11T18:13:34,663 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b9193c478b6b4517bbc382e916633b3b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b9193c478b6b4517bbc382e916633b3b 2024-11-11T18:13:34,675 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into b9193c478b6b4517bbc382e916633b3b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:34,675 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:34,675 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348814551; duration=0sec 2024-11-11T18:13:34,675 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:34,675 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:34,683 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:34,684 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/c7d870242bf4490ab14f042d537b5ac1 is 50, key is test_row_0/C:col10/1731348813909/Put/seqid=0 2024-11-11T18:13:34,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f1a0b2ee47f74c8597d4923de0a312b4 is 50, key is test_row_0/B:col10/1731348814559/Put/seqid=0 2024-11-11T18:13:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741894_1070 (size=12831) 2024-11-11T18:13:34,717 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/c7d870242bf4490ab14f042d537b5ac1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7d870242bf4490ab14f042d537b5ac1 2024-11-11T18:13:34,725 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into c7d870242bf4490ab14f042d537b5ac1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:34,725 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:34,726 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=13, startTime=1731348814552; duration=0sec 2024-11-11T18:13:34,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741893_1069 (size=12301) 2024-11-11T18:13:34,726 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:34,726 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:34,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f1a0b2ee47f74c8597d4923de0a312b4 2024-11-11T18:13:34,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/c7f7b71d9d81411ca4787b9cb3bd5bbd is 50, key is test_row_0/C:col10/1731348814559/Put/seqid=0 2024-11-11T18:13:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741895_1071 (size=12301) 2024-11-11T18:13:34,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348874754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348874765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348874765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348874766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348874766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:34,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:34,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:34,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348874965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348874974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348874975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348874975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348874975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:35,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:35,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T18:13:35,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/c7f7b71d9d81411ca4787b9cb3bd5bbd 2024-11-11T18:13:35,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6 2024-11-11T18:13:35,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6, entries=200, sequenceid=285, filesize=14.4 K 2024-11-11T18:13:35,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f1a0b2ee47f74c8597d4923de0a312b4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f1a0b2ee47f74c8597d4923de0a312b4 2024-11-11T18:13:35,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f1a0b2ee47f74c8597d4923de0a312b4, entries=150, sequenceid=285, filesize=12.0 K 2024-11-11T18:13:35,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/c7f7b71d9d81411ca4787b9cb3bd5bbd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7f7b71d9d81411ca4787b9cb3bd5bbd 2024-11-11T18:13:35,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7f7b71d9d81411ca4787b9cb3bd5bbd, entries=150, sequenceid=285, filesize=12.0 K 2024-11-11T18:13:35,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=140.89 KB/144270 for 2330eb7332d026bb8d820d13c904c2d4 in 621ms, sequenceid=285, compaction requested=false 2024-11-11T18:13:35,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:35,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T18:13:35,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:35,217 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-11T18:13:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:35,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/ce90803c397e48158e3b0c95fbb98a7e is 50, key is test_row_0/A:col10/1731348814635/Put/seqid=0 2024-11-11T18:13:35,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741896_1072 (size=12301) 2024-11-11T18:13:35,245 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/ce90803c397e48158e3b0c95fbb98a7e 2024-11-11T18:13:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/bbace6698b81411e87a7b3836d8a1c18 is 50, key is test_row_0/B:col10/1731348814635/Put/seqid=0 2024-11-11T18:13:35,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741897_1073 (size=12301) 2024-11-11T18:13:35,268 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/bbace6698b81411e87a7b3836d8a1c18 2024-11-11T18:13:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/25d6a33dadc340688f6ded0ec60ff666 is 50, key is test_row_0/C:col10/1731348814635/Put/seqid=0 2024-11-11T18:13:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:35,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348875298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741898_1074 (size=12301) 2024-11-11T18:13:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348875299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,311 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/25d6a33dadc340688f6ded0ec60ff666 2024-11-11T18:13:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348875301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348875302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348875303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/ce90803c397e48158e3b0c95fbb98a7e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/ce90803c397e48158e3b0c95fbb98a7e 2024-11-11T18:13:35,332 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/ce90803c397e48158e3b0c95fbb98a7e, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T18:13:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/bbace6698b81411e87a7b3836d8a1c18 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/bbace6698b81411e87a7b3836d8a1c18 2024-11-11T18:13:35,340 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/bbace6698b81411e87a7b3836d8a1c18, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T18:13:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/25d6a33dadc340688f6ded0ec60ff666 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/25d6a33dadc340688f6ded0ec60ff666 2024-11-11T18:13:35,349 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/25d6a33dadc340688f6ded0ec60ff666, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T18:13:35,350 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=53.67 KB/54960 for 2330eb7332d026bb8d820d13c904c2d4 in 133ms, sequenceid=321, compaction requested=true 2024-11-11T18:13:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:35,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T18:13:35,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T18:13:35,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T18:13:35,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3800 sec 2024-11-11T18:13:35,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3900 sec 2024-11-11T18:13:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:35,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:13:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:35,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:35,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b5e562fafa9a44278c9113dbefc49973 is 50, key is test_row_0/A:col10/1731348815416/Put/seqid=0 2024-11-11T18:13:35,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741899_1075 (size=14741) 2024-11-11T18:13:35,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b5e562fafa9a44278c9113dbefc49973 2024-11-11T18:13:35,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348815436, max timestamp=1731348815455Num uniques 859; 2024-11-11T18:13:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348815436, max timestamp=1731348815455Num uniques 859; 2024-11-11T18:13:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:13:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:13:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348815436, max timestamp=1731348815455Num uniques 859; 2024-11-11T18:13:35,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:13:35,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a7e0c93c1dd047d4b8c0d556f81af5d7 is 50, key is test_row_0/B:col10/1731348815416/Put/seqid=0 2024-11-11T18:13:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741900_1076 (size=12301) 2024-11-11T18:13:35,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a7e0c93c1dd047d4b8c0d556f81af5d7 2024-11-11T18:13:35,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348875489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348875493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348875493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348875496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1cf24690a1ba4c839ef2ee9f57625c9b is 50, key is test_row_0/C:col10/1731348815416/Put/seqid=0 2024-11-11T18:13:35,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348875503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741901_1077 (size=12301) 2024-11-11T18:13:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348875615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348875615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348875615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348875615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348875616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348875824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348875824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348875825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348875825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348875825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:35,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1cf24690a1ba4c839ef2ee9f57625c9b 2024-11-11T18:13:35,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b5e562fafa9a44278c9113dbefc49973 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b5e562fafa9a44278c9113dbefc49973 2024-11-11T18:13:35,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b5e562fafa9a44278c9113dbefc49973, entries=200, sequenceid=333, filesize=14.4 K 2024-11-11T18:13:35,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a7e0c93c1dd047d4b8c0d556f81af5d7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a7e0c93c1dd047d4b8c0d556f81af5d7 2024-11-11T18:13:35,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a7e0c93c1dd047d4b8c0d556f81af5d7, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T18:13:35,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1cf24690a1ba4c839ef2ee9f57625c9b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1cf24690a1ba4c839ef2ee9f57625c9b 2024-11-11T18:13:35,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1cf24690a1ba4c839ef2ee9f57625c9b, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T18:13:35,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 2330eb7332d026bb8d820d13c904c2d4 in 530ms, sequenceid=333, compaction requested=true 2024-11-11T18:13:35,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:35,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:35,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:35,947 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:35,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:35,948 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:35,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:35,950 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:35,950 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:35,950 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:35,950 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:35,950 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:35,950 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b9193c478b6b4517bbc382e916633b3b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/ce90803c397e48158e3b0c95fbb98a7e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b5e562fafa9a44278c9113dbefc49973] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=53.4 K 2024-11-11T18:13:35,950 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:35,950 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/91c620d174394d23afda3a08f959b999, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f1a0b2ee47f74c8597d4923de0a312b4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/bbace6698b81411e87a7b3836d8a1c18, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a7e0c93c1dd047d4b8c0d556f81af5d7] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=48.6 K 2024-11-11T18:13:35,951 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9193c478b6b4517bbc382e916633b3b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731348813909 2024-11-11T18:13:35,951 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c620d174394d23afda3a08f959b999, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731348813909 2024-11-11T18:13:35,951 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b7ac5fe4c7f45fdb4982ac008cbcdc6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731348814553 2024-11-11T18:13:35,951 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f1a0b2ee47f74c8597d4923de0a312b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731348814558 2024-11-11T18:13:35,951 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce90803c397e48158e3b0c95fbb98a7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731348814635 2024-11-11T18:13:35,952 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting bbace6698b81411e87a7b3836d8a1c18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731348814635 2024-11-11T18:13:35,952 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e562fafa9a44278c9113dbefc49973, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731348815297 2024-11-11T18:13:35,952 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a7e0c93c1dd047d4b8c0d556f81af5d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731348815298 2024-11-11T18:13:35,969 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:35,970 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/97ab27618c5d4eff8943d669753d769e is 50, key is test_row_0/A:col10/1731348815416/Put/seqid=0 2024-11-11T18:13:35,975 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:35,975 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ad2a35e4ff204b619a3b6cfa21fb4b16 is 50, key is test_row_0/B:col10/1731348815416/Put/seqid=0 2024-11-11T18:13:35,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741903_1079 (size=13051) 2024-11-11T18:13:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741902_1078 (size=13051) 2024-11-11T18:13:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T18:13:36,115 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:36,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T18:13:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T18:13:36,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:36,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:36,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:36,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T18:13:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:13:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:13:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:36,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:13:36,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/fbbf463a53224a619cb6ca08914f06ea is 50, key is test_row_0/A:col10/1731348815484/Put/seqid=0 2024-11-11T18:13:36,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741904_1080 (size=12301) 2024-11-11T18:13:36,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/fbbf463a53224a619cb6ca08914f06ea 2024-11-11T18:13:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348876149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348876150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348876153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348876160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348876161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/4450706b157140fc865bda84ac5c9aa9 is 50, key is test_row_0/B:col10/1731348815484/Put/seqid=0 2024-11-11T18:13:36,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741905_1081 (size=12301) 2024-11-11T18:13:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T18:13:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348876265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T18:13:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348876265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348876265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348876275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348876275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,401 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/ad2a35e4ff204b619a3b6cfa21fb4b16 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ad2a35e4ff204b619a3b6cfa21fb4b16 2024-11-11T18:13:36,401 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/97ab27618c5d4eff8943d669753d769e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/97ab27618c5d4eff8943d669753d769e 2024-11-11T18:13:36,413 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 97ab27618c5d4eff8943d669753d769e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:36,413 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into ad2a35e4ff204b619a3b6cfa21fb4b16(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:36,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:36,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:36,413 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=12, startTime=1731348815947; duration=0sec 2024-11-11T18:13:36,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:36,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:36,413 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=12, startTime=1731348815947; duration=0sec 2024-11-11T18:13:36,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:36,414 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:36,414 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:36,416 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:36,416 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:36,416 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,416 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7d870242bf4490ab14f042d537b5ac1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7f7b71d9d81411ca4787b9cb3bd5bbd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/25d6a33dadc340688f6ded0ec60ff666, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1cf24690a1ba4c839ef2ee9f57625c9b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=48.6 K 2024-11-11T18:13:36,418 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d870242bf4490ab14f042d537b5ac1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731348813909 2024-11-11T18:13:36,418 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f7b71d9d81411ca4787b9cb3bd5bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731348814558 2024-11-11T18:13:36,420 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d6a33dadc340688f6ded0ec60ff666, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731348814635 2024-11-11T18:13:36,420 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf24690a1ba4c839ef2ee9f57625c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731348815298 2024-11-11T18:13:36,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T18:13:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,434 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:36,434 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/38666a9dd1d447048e051dd099d6706c is 50, key is test_row_0/C:col10/1731348815416/Put/seqid=0 2024-11-11T18:13:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T18:13:36,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741906_1082 (size=13017) 2024-11-11T18:13:36,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348876474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348876485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348876484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348876485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348876494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T18:13:36,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:36,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:36,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/4450706b157140fc865bda84ac5c9aa9 2024-11-11T18:13:36,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/fb50520ed55c46ecb6854b6be5725ff7 is 50, key is test_row_0/C:col10/1731348815484/Put/seqid=0 2024-11-11T18:13:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741907_1083 (size=12301) 2024-11-11T18:13:36,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/fb50520ed55c46ecb6854b6be5725ff7 2024-11-11T18:13:36,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/fbbf463a53224a619cb6ca08914f06ea as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/fbbf463a53224a619cb6ca08914f06ea 2024-11-11T18:13:36,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/fbbf463a53224a619cb6ca08914f06ea, entries=150, sequenceid=371, filesize=12.0 K 2024-11-11T18:13:36,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/4450706b157140fc865bda84ac5c9aa9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/4450706b157140fc865bda84ac5c9aa9 2024-11-11T18:13:36,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/4450706b157140fc865bda84ac5c9aa9, entries=150, sequenceid=371, filesize=12.0 K 2024-11-11T18:13:36,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/fb50520ed55c46ecb6854b6be5725ff7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/fb50520ed55c46ecb6854b6be5725ff7 2024-11-11T18:13:36,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/fb50520ed55c46ecb6854b6be5725ff7, entries=150, sequenceid=371, filesize=12.0 K 2024-11-11T18:13:36,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 2330eb7332d026bb8d820d13c904c2d4 in 510ms, sequenceid=371, compaction requested=false 2024-11-11T18:13:36,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:36,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T18:13:36,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,736 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T18:13:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:36,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:36,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/243404e4875e4dfb9a86ef89418e4af7 is 50, key is test_row_0/A:col10/1731348816146/Put/seqid=0 2024-11-11T18:13:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T18:13:36,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741908_1084 (size=12301) 2024-11-11T18:13:36,749 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/243404e4875e4dfb9a86ef89418e4af7 2024-11-11T18:13:36,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/6e430caef278409cad27378e2b0ee656 is 50, key is test_row_0/B:col10/1731348816146/Put/seqid=0 2024-11-11T18:13:36,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741909_1085 (size=12301) 2024-11-11T18:13:36,771 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/6e430caef278409cad27378e2b0ee656 2024-11-11T18:13:36,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/954507c092394862a95e131fd4cad6c7 is 50, key is test_row_0/C:col10/1731348816146/Put/seqid=0 2024-11-11T18:13:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:36,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:36,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741910_1086 (size=12301) 2024-11-11T18:13:36,793 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/954507c092394862a95e131fd4cad6c7 2024-11-11T18:13:36,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/243404e4875e4dfb9a86ef89418e4af7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/243404e4875e4dfb9a86ef89418e4af7 2024-11-11T18:13:36,814 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/243404e4875e4dfb9a86ef89418e4af7, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T18:13:36,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/6e430caef278409cad27378e2b0ee656 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/6e430caef278409cad27378e2b0ee656 2024-11-11T18:13:36,824 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/6e430caef278409cad27378e2b0ee656, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T18:13:36,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/954507c092394862a95e131fd4cad6c7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/954507c092394862a95e131fd4cad6c7 2024-11-11T18:13:36,832 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/954507c092394862a95e131fd4cad6c7, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T18:13:36,833 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=107.34 KB/109920 for 2330eb7332d026bb8d820d13c904c2d4 in 97ms, sequenceid=381, compaction requested=true 2024-11-11T18:13:36,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:36,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:36,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T18:13:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T18:13:36,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T18:13:36,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:36,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:36,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:36,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:36,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:36,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:36,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T18:13:36,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 717 msec 2024-11-11T18:13:36,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 731 msec 2024-11-11T18:13:36,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5967604977d345a5a5dffb07c8ea51ba is 50, key is test_row_0/A:col10/1731348816831/Put/seqid=0 2024-11-11T18:13:36,855 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/38666a9dd1d447048e051dd099d6706c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/38666a9dd1d447048e051dd099d6706c 2024-11-11T18:13:36,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348876847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348876849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,863 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 38666a9dd1d447048e051dd099d6706c(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:36,863 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:36,863 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=12, startTime=1731348815948; duration=0sec 2024-11-11T18:13:36,863 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:36,863 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:36,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348876855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348876857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348876857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741911_1087 (size=17181) 2024-11-11T18:13:36,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5967604977d345a5a5dffb07c8ea51ba 2024-11-11T18:13:36,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a1523e22d0254bfa968dee7b8f59a94c is 50, key is test_row_0/B:col10/1731348816831/Put/seqid=0 2024-11-11T18:13:36,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741912_1088 (size=12301) 2024-11-11T18:13:36,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348876964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348876965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348876975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348876975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348876975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348877174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348877175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348877185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348877185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348877185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T18:13:37,254 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:37,255 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:37,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T18:13:37,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T18:13:37,257 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:37,258 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:37,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:37,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a1523e22d0254bfa968dee7b8f59a94c 2024-11-11T18:13:37,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/ce3b6785bb2141ccac7fd7d3472911c1 is 50, key is test_row_0/C:col10/1731348816831/Put/seqid=0 2024-11-11T18:13:37,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741913_1089 (size=12301) 2024-11-11T18:13:37,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/ce3b6785bb2141ccac7fd7d3472911c1 2024-11-11T18:13:37,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5967604977d345a5a5dffb07c8ea51ba as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5967604977d345a5a5dffb07c8ea51ba 2024-11-11T18:13:37,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5967604977d345a5a5dffb07c8ea51ba, entries=250, sequenceid=402, filesize=16.8 K 2024-11-11T18:13:37,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a1523e22d0254bfa968dee7b8f59a94c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1523e22d0254bfa968dee7b8f59a94c 2024-11-11T18:13:37,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1523e22d0254bfa968dee7b8f59a94c, entries=150, sequenceid=402, filesize=12.0 K 2024-11-11T18:13:37,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/ce3b6785bb2141ccac7fd7d3472911c1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ce3b6785bb2141ccac7fd7d3472911c1 2024-11-11T18:13:37,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ce3b6785bb2141ccac7fd7d3472911c1, entries=150, sequenceid=402, filesize=12.0 K 2024-11-11T18:13:37,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2330eb7332d026bb8d820d13c904c2d4 in 504ms, sequenceid=402, compaction requested=true 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:37,342 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:37,342 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:37,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:37,344 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:37,344 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:37,345 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:37,345 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:37,345 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:37,345 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:37,345 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ad2a35e4ff204b619a3b6cfa21fb4b16, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/4450706b157140fc865bda84ac5c9aa9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/6e430caef278409cad27378e2b0ee656, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1523e22d0254bfa968dee7b8f59a94c] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=48.8 K 2024-11-11T18:13:37,345 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/97ab27618c5d4eff8943d669753d769e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/fbbf463a53224a619cb6ca08914f06ea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/243404e4875e4dfb9a86ef89418e4af7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5967604977d345a5a5dffb07c8ea51ba] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=53.5 K 2024-11-11T18:13:37,345 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97ab27618c5d4eff8943d669753d769e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731348815298 2024-11-11T18:13:37,345 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2a35e4ff204b619a3b6cfa21fb4b16, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731348815298 2024-11-11T18:13:37,346 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbbf463a53224a619cb6ca08914f06ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731348815484 2024-11-11T18:13:37,346 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4450706b157140fc865bda84ac5c9aa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731348815484 2024-11-11T18:13:37,346 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243404e4875e4dfb9a86ef89418e4af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731348816146 2024-11-11T18:13:37,347 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e430caef278409cad27378e2b0ee656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731348816146 2024-11-11T18:13:37,347 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5967604977d345a5a5dffb07c8ea51ba, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731348816823 2024-11-11T18:13:37,348 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a1523e22d0254bfa968dee7b8f59a94c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731348816824 2024-11-11T18:13:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T18:13:37,371 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:37,373 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/60ab294ed3114e409ef0228e85e50f50 is 50, key is test_row_0/A:col10/1731348816831/Put/seqid=0 2024-11-11T18:13:37,379 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:37,379 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/2ae2f2db92ce4ef78e69b61308cd1eb6 is 50, key is test_row_0/B:col10/1731348816831/Put/seqid=0 2024-11-11T18:13:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741914_1090 (size=13187) 2024-11-11T18:13:37,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741915_1091 (size=13187) 2024-11-11T18:13:37,402 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/60ab294ed3114e409ef0228e85e50f50 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/60ab294ed3114e409ef0228e85e50f50 2024-11-11T18:13:37,410 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 60ab294ed3114e409ef0228e85e50f50(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:37,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:37,410 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=12, startTime=1731348817342; duration=0sec 2024-11-11T18:13:37,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:37,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:37,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T18:13:37,411 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:37,411 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/2ae2f2db92ce4ef78e69b61308cd1eb6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2ae2f2db92ce4ef78e69b61308cd1eb6 2024-11-11T18:13:37,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:37,412 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:13:37,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:37,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:37,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:37,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:37,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:37,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:37,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:37,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:37,414 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:37,414 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/38666a9dd1d447048e051dd099d6706c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/fb50520ed55c46ecb6854b6be5725ff7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/954507c092394862a95e131fd4cad6c7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ce3b6785bb2141ccac7fd7d3472911c1] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=48.8 K 2024-11-11T18:13:37,414 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38666a9dd1d447048e051dd099d6706c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731348815298 2024-11-11T18:13:37,415 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb50520ed55c46ecb6854b6be5725ff7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731348815484 2024-11-11T18:13:37,418 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954507c092394862a95e131fd4cad6c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731348816146 2024-11-11T18:13:37,418 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce3b6785bb2141ccac7fd7d3472911c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731348816824 2024-11-11T18:13:37,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c0eb0729618d4c31ab673b8bc8c77112 is 50, key is test_row_0/A:col10/1731348816855/Put/seqid=0 2024-11-11T18:13:37,423 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 2ae2f2db92ce4ef78e69b61308cd1eb6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:37,423 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:37,423 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=12, startTime=1731348817342; duration=0sec 2024-11-11T18:13:37,423 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:37,424 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:37,439 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:37,440 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/7b0fb2a50a484a8d8cec4144586d76b6 is 50, key is test_row_0/C:col10/1731348816831/Put/seqid=0 2024-11-11T18:13:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741916_1092 (size=12301) 2024-11-11T18:13:37,445 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c0eb0729618d4c31ab673b8bc8c77112 2024-11-11T18:13:37,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d71b197ef04e40eda75daacd9b590b8c is 50, key is test_row_0/B:col10/1731348816855/Put/seqid=0 2024-11-11T18:13:37,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741917_1093 (size=13153) 2024-11-11T18:13:37,468 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/7b0fb2a50a484a8d8cec4144586d76b6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/7b0fb2a50a484a8d8cec4144586d76b6 2024-11-11T18:13:37,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741918_1094 (size=12301) 2024-11-11T18:13:37,471 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d71b197ef04e40eda75daacd9b590b8c 2024-11-11T18:13:37,477 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 7b0fb2a50a484a8d8cec4144586d76b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:37,477 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:37,477 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=12, startTime=1731348817342; duration=0sec 2024-11-11T18:13:37,478 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:37,478 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d635e0faa27949f6ab9a2f5b33094a0c is 50, key is test_row_0/C:col10/1731348816855/Put/seqid=0 2024-11-11T18:13:37,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:37,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741919_1095 (size=12301) 2024-11-11T18:13:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731348817501, max timestamp=1731348817543Num uniques 859; 2024-11-11T18:13:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731348817501, max timestamp=1731348817543Num uniques 859; 2024-11-11T18:13:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731348817501, max timestamp=1731348817543Num uniques 859; 2024-11-11T18:13:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T18:13:37,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348877611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348877611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348877612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348877613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348877613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348877724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348877725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348877725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348877725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348877725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T18:13:37,893 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d635e0faa27949f6ab9a2f5b33094a0c 2024-11-11T18:13:37,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c0eb0729618d4c31ab673b8bc8c77112 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c0eb0729618d4c31ab673b8bc8c77112 2024-11-11T18:13:37,903 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c0eb0729618d4c31ab673b8bc8c77112, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T18:13:37,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/d71b197ef04e40eda75daacd9b590b8c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d71b197ef04e40eda75daacd9b590b8c 2024-11-11T18:13:37,910 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d71b197ef04e40eda75daacd9b590b8c, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T18:13:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d635e0faa27949f6ab9a2f5b33094a0c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d635e0faa27949f6ab9a2f5b33094a0c 2024-11-11T18:13:37,917 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d635e0faa27949f6ab9a2f5b33094a0c, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T18:13:37,919 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2330eb7332d026bb8d820d13c904c2d4 in 508ms, sequenceid=415, compaction requested=false 2024-11-11T18:13:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T18:13:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T18:13:37,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T18:13:37,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 662 msec 2024-11-11T18:13:37,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 668 msec 2024-11-11T18:13:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:37,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-11T18:13:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:37,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bcf208fda9534f7cb257d1fc5d85d241 is 50, key is test_row_0/A:col10/1731348817945/Put/seqid=0 2024-11-11T18:13:37,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741920_1096 (size=17181) 2024-11-11T18:13:37,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bcf208fda9534f7cb257d1fc5d85d241 2024-11-11T18:13:37,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348877958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348877958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348877958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348877960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348877960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:37,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/2e2efbcee0d24dcab285e1e219f54439 is 50, key is test_row_0/B:col10/1731348817945/Put/seqid=0 2024-11-11T18:13:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741921_1097 (size=12301) 2024-11-11T18:13:37,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/2e2efbcee0d24dcab285e1e219f54439 2024-11-11T18:13:37,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d96102086d4c4bec95f5cdaa0172ba5b is 50, key is test_row_0/C:col10/1731348817945/Put/seqid=0 2024-11-11T18:13:38,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741922_1098 (size=12301) 2024-11-11T18:13:38,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348878074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348878075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348878075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348878075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348878075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348878284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348878285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348878285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348878285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348878285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T18:13:38,395 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:38,395 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:38,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T18:13:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T18:13:38,397 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:38,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:38,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:38,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d96102086d4c4bec95f5cdaa0172ba5b 2024-11-11T18:13:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/bcf208fda9534f7cb257d1fc5d85d241 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bcf208fda9534f7cb257d1fc5d85d241 2024-11-11T18:13:38,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bcf208fda9534f7cb257d1fc5d85d241, entries=250, sequenceid=453, filesize=16.8 K 2024-11-11T18:13:38,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/2e2efbcee0d24dcab285e1e219f54439 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2e2efbcee0d24dcab285e1e219f54439 2024-11-11T18:13:38,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2e2efbcee0d24dcab285e1e219f54439, entries=150, sequenceid=453, filesize=12.0 K 2024-11-11T18:13:38,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d96102086d4c4bec95f5cdaa0172ba5b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d96102086d4c4bec95f5cdaa0172ba5b 2024-11-11T18:13:38,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d96102086d4c4bec95f5cdaa0172ba5b, entries=150, sequenceid=453, filesize=12.0 K 2024-11-11T18:13:38,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 2330eb7332d026bb8d820d13c904c2d4 in 487ms, sequenceid=453, compaction requested=true 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:38,434 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:38,434 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:38,436 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:38,436 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:38,436 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:38,436 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:38,436 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:38,436 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:38,436 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2ae2f2db92ce4ef78e69b61308cd1eb6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d71b197ef04e40eda75daacd9b590b8c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2e2efbcee0d24dcab285e1e219f54439] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=36.9 K 2024-11-11T18:13:38,436 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/60ab294ed3114e409ef0228e85e50f50, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c0eb0729618d4c31ab673b8bc8c77112, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bcf208fda9534f7cb257d1fc5d85d241] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=41.7 K 2024-11-11T18:13:38,437 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae2f2db92ce4ef78e69b61308cd1eb6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731348816824 2024-11-11T18:13:38,437 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ab294ed3114e409ef0228e85e50f50, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731348816824 2024-11-11T18:13:38,438 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d71b197ef04e40eda75daacd9b590b8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731348816846 2024-11-11T18:13:38,438 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0eb0729618d4c31ab673b8bc8c77112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731348816846 2024-11-11T18:13:38,438 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2efbcee0d24dcab285e1e219f54439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731348817611 2024-11-11T18:13:38,438 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf208fda9534f7cb257d1fc5d85d241, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731348817609 2024-11-11T18:13:38,452 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:38,453 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/13bbbcf31fb544e2a4f2f6d9e435fa96 is 50, key is test_row_0/B:col10/1731348817945/Put/seqid=0 2024-11-11T18:13:38,456 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:38,457 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5a68f5cfc6d640328c01a280777438a6 is 50, key is test_row_0/A:col10/1731348817945/Put/seqid=0 2024-11-11T18:13:38,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741924_1100 (size=13289) 2024-11-11T18:13:38,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741923_1099 (size=13289) 2024-11-11T18:13:38,479 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/5a68f5cfc6d640328c01a280777438a6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5a68f5cfc6d640328c01a280777438a6 2024-11-11T18:13:38,488 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/13bbbcf31fb544e2a4f2f6d9e435fa96 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/13bbbcf31fb544e2a4f2f6d9e435fa96 2024-11-11T18:13:38,493 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 5a68f5cfc6d640328c01a280777438a6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:38,493 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:38,493 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348818434; duration=0sec 2024-11-11T18:13:38,493 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:38,493 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:38,493 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:38,494 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:38,494 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:38,494 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:38,495 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/7b0fb2a50a484a8d8cec4144586d76b6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d635e0faa27949f6ab9a2f5b33094a0c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d96102086d4c4bec95f5cdaa0172ba5b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=36.9 K 2024-11-11T18:13:38,495 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0fb2a50a484a8d8cec4144586d76b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731348816824 2024-11-11T18:13:38,495 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 13bbbcf31fb544e2a4f2f6d9e435fa96(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:38,495 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:38,495 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d635e0faa27949f6ab9a2f5b33094a0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731348816846 2024-11-11T18:13:38,496 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348818434; duration=0sec 2024-11-11T18:13:38,496 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:38,496 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:38,496 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d96102086d4c4bec95f5cdaa0172ba5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731348817611 2024-11-11T18:13:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T18:13:38,505 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:38,505 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1c1fe39f45af4690b1cc53e89a1c4ef3 is 50, key is test_row_0/C:col10/1731348817945/Put/seqid=0 2024-11-11T18:13:38,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741925_1101 (size=13255) 2024-11-11T18:13:38,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T18:13:38,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:38,552 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c822fa7073ef4065b8c69fb2da9fce9f is 50, key is test_row_0/A:col10/1731348817956/Put/seqid=0 2024-11-11T18:13:38,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741926_1102 (size=12301) 2024-11-11T18:13:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:38,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:38,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348818600, max timestamp=1731348818624Num uniques 859; 2024-11-11T18:13:38,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348818600, max timestamp=1731348818624Num uniques 859; 2024-11-11T18:13:38,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348818600, max timestamp=1731348818624Num uniques 859; 2024-11-11T18:13:38,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:38,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:38,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348878649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348878649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348878651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348878651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348878652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T18:13:38,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348878765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348878765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348878765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348878765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348878765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,918 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1c1fe39f45af4690b1cc53e89a1c4ef3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c1fe39f45af4690b1cc53e89a1c4ef3 2024-11-11T18:13:38,925 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 1c1fe39f45af4690b1cc53e89a1c4ef3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:38,925 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:38,925 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=13, startTime=1731348818434; duration=0sec 2024-11-11T18:13:38,925 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:38,925 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:38,965 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c822fa7073ef4065b8c69fb2da9fce9f 2024-11-11T18:13:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a45da1308b3c4453adf02e5fe66cc1d1 is 50, key is test_row_0/B:col10/1731348817956/Put/seqid=0 2024-11-11T18:13:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348878976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348878976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348878977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348878978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348878977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:38,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741927_1103 (size=12301) 2024-11-11T18:13:38,997 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a45da1308b3c4453adf02e5fe66cc1d1 2024-11-11T18:13:39,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d97bca450c9e4464b82570837ab4ad3b is 50, key is test_row_0/C:col10/1731348817956/Put/seqid=0 2024-11-11T18:13:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T18:13:39,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741928_1104 (size=12301) 2024-11-11T18:13:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348879285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348879285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348879285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348879286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348879286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,455 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d97bca450c9e4464b82570837ab4ad3b 2024-11-11T18:13:39,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c822fa7073ef4065b8c69fb2da9fce9f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c822fa7073ef4065b8c69fb2da9fce9f 2024-11-11T18:13:39,466 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c822fa7073ef4065b8c69fb2da9fce9f, entries=150, sequenceid=465, filesize=12.0 K 2024-11-11T18:13:39,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/a45da1308b3c4453adf02e5fe66cc1d1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a45da1308b3c4453adf02e5fe66cc1d1 2024-11-11T18:13:39,473 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a45da1308b3c4453adf02e5fe66cc1d1, entries=150, sequenceid=465, filesize=12.0 K 2024-11-11T18:13:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/d97bca450c9e4464b82570837ab4ad3b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d97bca450c9e4464b82570837ab4ad3b 2024-11-11T18:13:39,480 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d97bca450c9e4464b82570837ab4ad3b, entries=150, sequenceid=465, filesize=12.0 K 2024-11-11T18:13:39,481 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2330eb7332d026bb8d820d13c904c2d4 in 929ms, sequenceid=465, compaction requested=false 2024-11-11T18:13:39,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:39,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:39,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T18:13:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T18:13:39,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T18:13:39,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0850 sec 2024-11-11T18:13:39,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0900 sec 2024-11-11T18:13:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T18:13:39,535 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:39,535 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T18:13:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T18:13:39,538 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:39,539 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:39,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T18:13:39,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T18:13:39,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:39,691 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-11T18:13:39,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:39,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:39,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:39,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:39,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:39,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/f43ea31778ea45dcb1a5a826febd88d4 is 50, key is test_row_0/A:col10/1731348818646/Put/seqid=0 2024-11-11T18:13:39,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741929_1105 (size=12301) 2024-11-11T18:13:39,711 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/f43ea31778ea45dcb1a5a826febd88d4 2024-11-11T18:13:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/707a665158c442d399aa9891640c4072 is 50, key is test_row_0/B:col10/1731348818646/Put/seqid=0 2024-11-11T18:13:39,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741930_1106 (size=12301) 2024-11-11T18:13:39,735 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/707a665158c442d399aa9891640c4072 2024-11-11T18:13:39,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/cad9ed4d32b5426eb74d61cbfb66a690 is 50, key is test_row_0/C:col10/1731348818646/Put/seqid=0 2024-11-11T18:13:39,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741931_1107 (size=12301) 2024-11-11T18:13:39,759 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/cad9ed4d32b5426eb74d61cbfb66a690 2024-11-11T18:13:39,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/f43ea31778ea45dcb1a5a826febd88d4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f43ea31778ea45dcb1a5a826febd88d4 2024-11-11T18:13:39,776 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f43ea31778ea45dcb1a5a826febd88d4, entries=150, sequenceid=501, filesize=12.0 K 2024-11-11T18:13:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/707a665158c442d399aa9891640c4072 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/707a665158c442d399aa9891640c4072 2024-11-11T18:13:39,789 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/707a665158c442d399aa9891640c4072, entries=150, sequenceid=501, filesize=12.0 K 2024-11-11T18:13:39,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/cad9ed4d32b5426eb74d61cbfb66a690 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cad9ed4d32b5426eb74d61cbfb66a690 2024-11-11T18:13:39,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:39,807 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cad9ed4d32b5426eb74d61cbfb66a690, entries=150, sequenceid=501, filesize=12.0 K 2024-11-11T18:13:39,809 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=33.54 KB/34350 for 2330eb7332d026bb8d820d13c904c2d4 in 118ms, sequenceid=501, compaction requested=true 2024-11-11T18:13:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T18:13:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T18:13:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:39,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:13:39,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:39,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:39,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:39,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:39,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:39,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:39,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T18:13:39,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 273 msec 2024-11-11T18:13:39,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 282 msec 2024-11-11T18:13:39,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a0f4c367ff664b6abea8e25bd26abdac is 50, key is test_row_0/A:col10/1731348819799/Put/seqid=0 2024-11-11T18:13:39,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741932_1108 (size=19621) 2024-11-11T18:13:39,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a0f4c367ff664b6abea8e25bd26abdac 2024-11-11T18:13:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T18:13:39,855 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:39,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T18:13:39,858 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T18:13:39,859 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:39,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:39,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348819841, max timestamp=1731348819854Num uniques 859; 2024-11-11T18:13:39,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:39,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348819841, max timestamp=1731348819854Num uniques 859; 2024-11-11T18:13:39,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:39,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348819841, max timestamp=1731348819854Num uniques 859; 2024-11-11T18:13:39,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:39,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/135baafef2f14a4a80383ed6ceae3a2c is 50, key is test_row_0/B:col10/1731348819799/Put/seqid=0 2024-11-11T18:13:39,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348879897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348879904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348879906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348879907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348879908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:39,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:39,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741933_1109 (size=12301) 2024-11-11T18:13:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T18:13:40,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T18:13:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348880015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348880029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348880029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348880029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348880029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T18:13:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:40,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T18:13:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348880234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348880245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348880245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348880245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348880245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T18:13:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:40,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/135baafef2f14a4a80383ed6ceae3a2c 2024-11-11T18:13:40,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/ae8181a614224cb1b5fb8041437c7a53 is 50, key is test_row_0/C:col10/1731348819799/Put/seqid=0 2024-11-11T18:13:40,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741934_1110 (size=12301) 2024-11-11T18:13:40,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/ae8181a614224cb1b5fb8041437c7a53 2024-11-11T18:13:40,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a0f4c367ff664b6abea8e25bd26abdac as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a0f4c367ff664b6abea8e25bd26abdac 2024-11-11T18:13:40,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a0f4c367ff664b6abea8e25bd26abdac, entries=300, sequenceid=512, filesize=19.2 K 2024-11-11T18:13:40,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/135baafef2f14a4a80383ed6ceae3a2c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/135baafef2f14a4a80383ed6ceae3a2c 2024-11-11T18:13:40,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/135baafef2f14a4a80383ed6ceae3a2c, entries=150, sequenceid=512, filesize=12.0 K 2024-11-11T18:13:40,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/ae8181a614224cb1b5fb8041437c7a53 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ae8181a614224cb1b5fb8041437c7a53 2024-11-11T18:13:40,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ae8181a614224cb1b5fb8041437c7a53, entries=150, sequenceid=512, filesize=12.0 K 2024-11-11T18:13:40,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2330eb7332d026bb8d820d13c904c2d4 in 612ms, sequenceid=512, compaction requested=true 2024-11-11T18:13:40,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:40,423 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:40,423 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:40,424 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:40,425 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:40,425 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:40,425 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,425 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:40,425 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,425 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5a68f5cfc6d640328c01a280777438a6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c822fa7073ef4065b8c69fb2da9fce9f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f43ea31778ea45dcb1a5a826febd88d4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a0f4c367ff664b6abea8e25bd26abdac] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=56.2 K 2024-11-11T18:13:40,425 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/13bbbcf31fb544e2a4f2f6d9e435fa96, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a45da1308b3c4453adf02e5fe66cc1d1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/707a665158c442d399aa9891640c4072, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/135baafef2f14a4a80383ed6ceae3a2c] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=49.0 K 2024-11-11T18:13:40,425 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a68f5cfc6d640328c01a280777438a6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731348817611 2024-11-11T18:13:40,426 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c822fa7073ef4065b8c69fb2da9fce9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731348817955 2024-11-11T18:13:40,427 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f43ea31778ea45dcb1a5a826febd88d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731348818646 2024-11-11T18:13:40,427 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bbbcf31fb544e2a4f2f6d9e435fa96, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731348817611 2024-11-11T18:13:40,429 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f4c367ff664b6abea8e25bd26abdac, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731348819799 2024-11-11T18:13:40,429 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a45da1308b3c4453adf02e5fe66cc1d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731348817955 2024-11-11T18:13:40,430 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 707a665158c442d399aa9891640c4072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731348818646 2024-11-11T18:13:40,432 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 135baafef2f14a4a80383ed6ceae3a2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731348819799 2024-11-11T18:13:40,449 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:40,450 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4ffa37ef47884dd0b13526602a1c5a63 is 50, key is test_row_0/A:col10/1731348819799/Put/seqid=0 2024-11-11T18:13:40,459 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:40,460 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/7470b38d9c9b4678a85136152efbca97 is 50, key is test_row_0/B:col10/1731348819799/Put/seqid=0 2024-11-11T18:13:40,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741935_1111 (size=13425) 2024-11-11T18:13:40,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T18:13:40,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,481 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T18:13:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:40,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T18:13:40,489 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4ffa37ef47884dd0b13526602a1c5a63 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4ffa37ef47884dd0b13526602a1c5a63 2024-11-11T18:13:40,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741936_1112 (size=13425) 2024-11-11T18:13:40,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b6b9a2199d7c4c5e94e81597af675637 is 50, key is test_row_0/A:col10/1731348819904/Put/seqid=0 2024-11-11T18:13:40,501 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 4ffa37ef47884dd0b13526602a1c5a63(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:40,501 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:40,501 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=12, startTime=1731348820423; duration=0sec 2024-11-11T18:13:40,502 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:40,502 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:40,502 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:40,507 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:40,507 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:40,507 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:40,507 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c1fe39f45af4690b1cc53e89a1c4ef3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d97bca450c9e4464b82570837ab4ad3b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cad9ed4d32b5426eb74d61cbfb66a690, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ae8181a614224cb1b5fb8041437c7a53] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=49.0 K 2024-11-11T18:13:40,509 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c1fe39f45af4690b1cc53e89a1c4ef3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731348817611 2024-11-11T18:13:40,510 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d97bca450c9e4464b82570837ab4ad3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731348817955 2024-11-11T18:13:40,510 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad9ed4d32b5426eb74d61cbfb66a690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731348818646 2024-11-11T18:13:40,511 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae8181a614224cb1b5fb8041437c7a53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731348819799 2024-11-11T18:13:40,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741937_1113 (size=12301) 2024-11-11T18:13:40,543 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:40,544 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/24422fef92764f58b3c6724ee03b645d is 50, key is test_row_0/C:col10/1731348819799/Put/seqid=0 2024-11-11T18:13:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:40,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348880562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348880563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741938_1114 (size=13391) 2024-11-11T18:13:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348880564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348880570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348880571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348880675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348880675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348880675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348880685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348880685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348880895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348880895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348880896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348880897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348880898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:40,903 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/7470b38d9c9b4678a85136152efbca97 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/7470b38d9c9b4678a85136152efbca97 2024-11-11T18:13:40,916 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b6b9a2199d7c4c5e94e81597af675637 2024-11-11T18:13:40,919 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 7470b38d9c9b4678a85136152efbca97(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:40,919 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:40,919 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=12, startTime=1731348820423; duration=0sec 2024-11-11T18:13:40,920 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:40,920 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3ceeb4c364774581a5aa14b08952dccd is 50, key is test_row_0/B:col10/1731348819904/Put/seqid=0 2024-11-11T18:13:40,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741939_1115 (size=12301) 2024-11-11T18:13:40,984 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/24422fef92764f58b3c6724ee03b645d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/24422fef92764f58b3c6724ee03b645d 2024-11-11T18:13:40,992 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 24422fef92764f58b3c6724ee03b645d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:40,992 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:40,992 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=12, startTime=1731348820423; duration=0sec 2024-11-11T18:13:40,993 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:40,993 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T18:13:41,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348881205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348881205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348881205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348881206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348881215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,353 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3ceeb4c364774581a5aa14b08952dccd 2024-11-11T18:13:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/52ad74d7003948ce9040401841dad75a is 50, key is test_row_0/C:col10/1731348819904/Put/seqid=0 2024-11-11T18:13:41,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741940_1116 (size=12301) 2024-11-11T18:13:41,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348881714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348881714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348881714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348881725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348881745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:41,805 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/52ad74d7003948ce9040401841dad75a 2024-11-11T18:13:41,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b6b9a2199d7c4c5e94e81597af675637 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b6b9a2199d7c4c5e94e81597af675637 2024-11-11T18:13:41,825 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b6b9a2199d7c4c5e94e81597af675637, entries=150, sequenceid=549, filesize=12.0 K 2024-11-11T18:13:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3ceeb4c364774581a5aa14b08952dccd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3ceeb4c364774581a5aa14b08952dccd 2024-11-11T18:13:41,844 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3ceeb4c364774581a5aa14b08952dccd, entries=150, sequenceid=549, filesize=12.0 K 2024-11-11T18:13:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/52ad74d7003948ce9040401841dad75a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/52ad74d7003948ce9040401841dad75a 2024-11-11T18:13:41,856 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/52ad74d7003948ce9040401841dad75a, entries=150, sequenceid=549, filesize=12.0 K 2024-11-11T18:13:41,863 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 2330eb7332d026bb8d820d13c904c2d4 in 1381ms, sequenceid=549, compaction requested=false 2024-11-11T18:13:41,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:41,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:41,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T18:13:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T18:13:41,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T18:13:41,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-11T18:13:41,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.0120 sec 2024-11-11T18:13:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T18:13:42,009 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:42,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T18:13:42,013 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T18:13:42,015 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:42,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T18:13:42,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T18:13:42,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:42,168 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:13:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:42,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/68dc44b80486461493ab65df790eeb1e is 50, key is test_row_0/A:col10/1731348820569/Put/seqid=0 2024-11-11T18:13:42,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741941_1117 (size=12301) 2024-11-11T18:13:42,228 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/68dc44b80486461493ab65df790eeb1e 2024-11-11T18:13:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/8ae3c3cf1f2743ba94958f2da251e5d2 is 50, key is test_row_0/B:col10/1731348820569/Put/seqid=0 2024-11-11T18:13:42,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741942_1118 (size=12301) 2024-11-11T18:13:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T18:13:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T18:13:42,664 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/8ae3c3cf1f2743ba94958f2da251e5d2 2024-11-11T18:13:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/f74259b2b0944fd0aeda96363cfc1fe0 is 50, key is test_row_0/C:col10/1731348820569/Put/seqid=0 2024-11-11T18:13:42,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741943_1119 (size=12301) 2024-11-11T18:13:42,682 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/f74259b2b0944fd0aeda96363cfc1fe0 2024-11-11T18:13:42,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/68dc44b80486461493ab65df790eeb1e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/68dc44b80486461493ab65df790eeb1e 2024-11-11T18:13:42,701 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/68dc44b80486461493ab65df790eeb1e, entries=150, sequenceid=562, filesize=12.0 K 2024-11-11T18:13:42,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/8ae3c3cf1f2743ba94958f2da251e5d2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8ae3c3cf1f2743ba94958f2da251e5d2 2024-11-11T18:13:42,709 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8ae3c3cf1f2743ba94958f2da251e5d2, entries=150, sequenceid=562, filesize=12.0 K 2024-11-11T18:13:42,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/f74259b2b0944fd0aeda96363cfc1fe0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/f74259b2b0944fd0aeda96363cfc1fe0 2024-11-11T18:13:42,719 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/f74259b2b0944fd0aeda96363cfc1fe0, entries=150, sequenceid=562, filesize=12.0 K 2024-11-11T18:13:42,720 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 2330eb7332d026bb8d820d13c904c2d4 in 552ms, sequenceid=562, compaction requested=true 2024-11-11T18:13:42,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:42,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:42,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T18:13:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T18:13:42,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T18:13:42,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 707 msec 2024-11-11T18:13:42,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 716 msec 2024-11-11T18:13:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:42,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:13:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:42,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:42,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:42,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:42,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:42,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/6c212eeefe8f4b538019be6a5897212e is 50, key is test_row_0/A:col10/1731348822738/Put/seqid=0 2024-11-11T18:13:42,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741944_1120 (size=17181) 2024-11-11T18:13:42,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/6c212eeefe8f4b538019be6a5897212e 2024-11-11T18:13:42,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348822782, max timestamp=1731348822801Num uniques 859; 2024-11-11T18:13:42,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:42,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348822782, max timestamp=1731348822801Num uniques 859; 2024-11-11T18:13:42,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:42,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348822782, max timestamp=1731348822801Num uniques 859; 2024-11-11T18:13:42,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:13:42,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/90ee7a08c09642b18dc61a0bcbfcb3ee is 50, key is test_row_0/B:col10/1731348822738/Put/seqid=0 2024-11-11T18:13:42,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741945_1121 (size=12301) 2024-11-11T18:13:42,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348882836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348882841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348882843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348882843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348882844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348882954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348882955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348882955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348882955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348882955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:42,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:42,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T18:13:43,155 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:43,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-11T18:13:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T18:13:43,158 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:43,160 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:43,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348883165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348883165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348883165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348883165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348883167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/90ee7a08c09642b18dc61a0bcbfcb3ee 2024-11-11T18:13:43,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/584d6c045bc94a6298c19b1cd0071f3d is 50, key is test_row_0/C:col10/1731348822738/Put/seqid=0 2024-11-11T18:13:43,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741946_1122 (size=12301) 2024-11-11T18:13:43,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/584d6c045bc94a6298c19b1cd0071f3d 2024-11-11T18:13:43,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/6c212eeefe8f4b538019be6a5897212e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6c212eeefe8f4b538019be6a5897212e 2024-11-11T18:13:43,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6c212eeefe8f4b538019be6a5897212e, entries=250, sequenceid=572, filesize=16.8 K 2024-11-11T18:13:43,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/90ee7a08c09642b18dc61a0bcbfcb3ee as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/90ee7a08c09642b18dc61a0bcbfcb3ee 2024-11-11T18:13:43,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/90ee7a08c09642b18dc61a0bcbfcb3ee, entries=150, sequenceid=572, filesize=12.0 K 2024-11-11T18:13:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T18:13:43,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/584d6c045bc94a6298c19b1cd0071f3d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/584d6c045bc94a6298c19b1cd0071f3d 2024-11-11T18:13:43,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/584d6c045bc94a6298c19b1cd0071f3d, entries=150, sequenceid=572, filesize=12.0 K 2024-11-11T18:13:43,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2330eb7332d026bb8d820d13c904c2d4 in 534ms, sequenceid=572, compaction requested=true 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:43,280 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:43,280 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:43,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:43,282 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:43,282 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:43,282 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,282 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/7470b38d9c9b4678a85136152efbca97, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3ceeb4c364774581a5aa14b08952dccd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8ae3c3cf1f2743ba94958f2da251e5d2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/90ee7a08c09642b18dc61a0bcbfcb3ee] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=49.1 K 2024-11-11T18:13:43,282 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55208 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:43,283 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:43,283 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,283 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4ffa37ef47884dd0b13526602a1c5a63, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b6b9a2199d7c4c5e94e81597af675637, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/68dc44b80486461493ab65df790eeb1e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6c212eeefe8f4b538019be6a5897212e] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=53.9 K 2024-11-11T18:13:43,283 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7470b38d9c9b4678a85136152efbca97, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731348819799 2024-11-11T18:13:43,283 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffa37ef47884dd0b13526602a1c5a63, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731348819799 2024-11-11T18:13:43,284 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ceeb4c364774581a5aa14b08952dccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731348819902 2024-11-11T18:13:43,284 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b9a2199d7c4c5e94e81597af675637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731348819902 2024-11-11T18:13:43,284 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae3c3cf1f2743ba94958f2da251e5d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731348820558 2024-11-11T18:13:43,285 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68dc44b80486461493ab65df790eeb1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731348820558 2024-11-11T18:13:43,285 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ee7a08c09642b18dc61a0bcbfcb3ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731348822738 2024-11-11T18:13:43,285 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c212eeefe8f4b538019be6a5897212e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731348822727 2024-11-11T18:13:43,311 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#110 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:43,312 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/d4cad6206f2e43cd983b189fb0d34239 is 50, key is test_row_0/A:col10/1731348822738/Put/seqid=0 2024-11-11T18:13:43,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,319 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:43,320 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:43,321 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3987e8fbfd974d8fbeb151253548e01a is 50, key is test_row_0/B:col10/1731348822738/Put/seqid=0 2024-11-11T18:13:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/98778a1497dc43cfb45ebbf0e6d4d633 is 50, key is test_row_0/A:col10/1731348822842/Put/seqid=0 2024-11-11T18:13:43,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741947_1123 (size=13561) 2024-11-11T18:13:43,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741948_1124 (size=13561) 2024-11-11T18:13:43,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741949_1125 (size=12301) 2024-11-11T18:13:43,370 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/98778a1497dc43cfb45ebbf0e6d4d633 2024-11-11T18:13:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/11819673050d4f099dc85a5afd1a62a1 is 50, key is test_row_0/B:col10/1731348822842/Put/seqid=0 2024-11-11T18:13:43,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741950_1126 (size=12301) 2024-11-11T18:13:43,410 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/11819673050d4f099dc85a5afd1a62a1 2024-11-11T18:13:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/15a4c208c43043c29672986254b453c1 is 50, key is test_row_0/C:col10/1731348822842/Put/seqid=0 2024-11-11T18:13:43,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741951_1127 (size=12301) 2024-11-11T18:13:43,461 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/15a4c208c43043c29672986254b453c1 2024-11-11T18:13:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T18:13:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/98778a1497dc43cfb45ebbf0e6d4d633 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98778a1497dc43cfb45ebbf0e6d4d633 2024-11-11T18:13:43,486 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98778a1497dc43cfb45ebbf0e6d4d633, entries=150, sequenceid=607, filesize=12.0 K 2024-11-11T18:13:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/11819673050d4f099dc85a5afd1a62a1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/11819673050d4f099dc85a5afd1a62a1 2024-11-11T18:13:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:43,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:43,495 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/11819673050d4f099dc85a5afd1a62a1, entries=150, sequenceid=607, filesize=12.0 K 2024-11-11T18:13:43,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/15a4c208c43043c29672986254b453c1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/15a4c208c43043c29672986254b453c1 2024-11-11T18:13:43,504 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/15a4c208c43043c29672986254b453c1, entries=150, sequenceid=607, filesize=12.0 K 2024-11-11T18:13:43,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348883498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,505 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 2330eb7332d026bb8d820d13c904c2d4 in 186ms, sequenceid=607, compaction requested=true 2024-11-11T18:13:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-11T18:13:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-11T18:13:43,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:13:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:43,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:43,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T18:13:43,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 349 msec 2024-11-11T18:13:43,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 358 msec 2024-11-11T18:13:43,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/e1ba2cd4ce6842959665b117ec5fd63e is 50, key is test_row_0/A:col10/1731348823502/Put/seqid=0 2024-11-11T18:13:43,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741952_1128 (size=17181) 2024-11-11T18:13:43,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348883542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348883546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348883546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348883550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348883614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348883655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348883655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348883655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348883664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,764 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/d4cad6206f2e43cd983b189fb0d34239 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/d4cad6206f2e43cd983b189fb0d34239 2024-11-11T18:13:43,782 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into d4cad6206f2e43cd983b189fb0d34239(size=13.2 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:43,782 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:43,782 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=12, startTime=1731348823280; duration=0sec 2024-11-11T18:13:43,783 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:43,783 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:43,783 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T18:13:43,783 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3987e8fbfd974d8fbeb151253548e01a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3987e8fbfd974d8fbeb151253548e01a 2024-11-11T18:13:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T18:13:43,786 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:43,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-11T18:13:43,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:43,789 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62595 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T18:13:43,789 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:43,789 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,789 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/24422fef92764f58b3c6724ee03b645d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/52ad74d7003948ce9040401841dad75a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/f74259b2b0944fd0aeda96363cfc1fe0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/584d6c045bc94a6298c19b1cd0071f3d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/15a4c208c43043c29672986254b453c1] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=61.1 K 2024-11-11T18:13:43,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:43,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T18:13:43,792 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24422fef92764f58b3c6724ee03b645d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731348819799 2024-11-11T18:13:43,792 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52ad74d7003948ce9040401841dad75a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731348819902 2024-11-11T18:13:43,793 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f74259b2b0944fd0aeda96363cfc1fe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731348820558 2024-11-11T18:13:43,796 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584d6c045bc94a6298c19b1cd0071f3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731348822738 2024-11-11T18:13:43,797 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 3987e8fbfd974d8fbeb151253548e01a(size=13.2 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:43,797 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:43,797 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=12, startTime=1731348823280; duration=0sec 2024-11-11T18:13:43,797 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a4c208c43043c29672986254b453c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731348822840 2024-11-11T18:13:43,797 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:43,797 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:43,821 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#116 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:43,822 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1c9c83c2af9649aea40e9c13b4ca6767 is 50, key is test_row_0/C:col10/1731348822842/Put/seqid=0 2024-11-11T18:13:43,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348883824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741953_1129 (size=13561) 2024-11-11T18:13:43,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348883865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348883865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348883865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348883877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T18:13:43,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/e1ba2cd4ce6842959665b117ec5fd63e 2024-11-11T18:13:43,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-11T18:13:43,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:43,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:43,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:43,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:43,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f098a694b45c4446bfa36225c2cb14c5 is 50, key is test_row_0/B:col10/1731348823502/Put/seqid=0 2024-11-11T18:13:43,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741954_1130 (size=12301) 2024-11-11T18:13:43,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f098a694b45c4446bfa36225c2cb14c5 2024-11-11T18:13:44,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/8cd1e3bd1dff4b74997a28a8371cc6e6 is 50, key is test_row_0/C:col10/1731348823502/Put/seqid=0 2024-11-11T18:13:44,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741955_1131 (size=12301) 2024-11-11T18:13:44,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-11T18:13:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T18:13:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348884136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348884175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348884176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348884185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348884185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,255 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/1c9c83c2af9649aea40e9c13b4ca6767 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c9c83c2af9649aea40e9c13b4ca6767 2024-11-11T18:13:44,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-11T18:13:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,286 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 1c9c83c2af9649aea40e9c13b4ca6767(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:44,286 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:44,286 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=11, startTime=1731348823280; duration=0sec 2024-11-11T18:13:44,287 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:44,287 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:44,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=619 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/8cd1e3bd1dff4b74997a28a8371cc6e6 2024-11-11T18:13:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T18:13:44,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-11T18:13:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:44,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/e1ba2cd4ce6842959665b117ec5fd63e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/e1ba2cd4ce6842959665b117ec5fd63e 2024-11-11T18:13:44,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/e1ba2cd4ce6842959665b117ec5fd63e, entries=250, sequenceid=619, filesize=16.8 K 2024-11-11T18:13:44,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/f098a694b45c4446bfa36225c2cb14c5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f098a694b45c4446bfa36225c2cb14c5 2024-11-11T18:13:44,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f098a694b45c4446bfa36225c2cb14c5, entries=150, sequenceid=619, filesize=12.0 K 2024-11-11T18:13:44,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/8cd1e3bd1dff4b74997a28a8371cc6e6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/8cd1e3bd1dff4b74997a28a8371cc6e6 2024-11-11T18:13:44,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/8cd1e3bd1dff4b74997a28a8371cc6e6, entries=150, sequenceid=619, filesize=12.0 K 2024-11-11T18:13:44,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2330eb7332d026bb8d820d13c904c2d4 in 933ms, sequenceid=619, compaction requested=true 2024-11-11T18:13:44,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:44,441 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:44,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:44,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:44,442 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:44,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:44,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:44,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:44,442 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:44,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:44,443 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:44,443 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,443 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/d4cad6206f2e43cd983b189fb0d34239, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98778a1497dc43cfb45ebbf0e6d4d633, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/e1ba2cd4ce6842959665b117ec5fd63e] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=42.0 K 2024-11-11T18:13:44,443 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4cad6206f2e43cd983b189fb0d34239, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731348822738 2024-11-11T18:13:44,444 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98778a1497dc43cfb45ebbf0e6d4d633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731348822840 2024-11-11T18:13:44,444 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:44,444 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:44,444 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,444 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3987e8fbfd974d8fbeb151253548e01a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/11819673050d4f099dc85a5afd1a62a1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f098a694b45c4446bfa36225c2cb14c5] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=37.3 K 2024-11-11T18:13:44,445 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ba2cd4ce6842959665b117ec5fd63e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1731348823487 2024-11-11T18:13:44,445 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3987e8fbfd974d8fbeb151253548e01a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1731348822738 2024-11-11T18:13:44,446 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 11819673050d4f099dc85a5afd1a62a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731348822840 2024-11-11T18:13:44,447 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f098a694b45c4446bfa36225c2cb14c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1731348823502 2024-11-11T18:13:44,466 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:44,466 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/9081f519fe614af9a1e46bccfe49ab55 is 50, key is test_row_0/B:col10/1731348823502/Put/seqid=0 2024-11-11T18:13:44,471 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:44,472 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/98b474e5a34c431989e96e139bc31247 is 50, key is test_row_0/A:col10/1731348823502/Put/seqid=0 2024-11-11T18:13:44,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741956_1132 (size=13663) 2024-11-11T18:13:44,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741957_1133 (size=13663) 2024-11-11T18:13:44,493 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/9081f519fe614af9a1e46bccfe49ab55 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/9081f519fe614af9a1e46bccfe49ab55 2024-11-11T18:13:44,501 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 9081f519fe614af9a1e46bccfe49ab55(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:44,501 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:44,501 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348824442; duration=0sec 2024-11-11T18:13:44,501 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:44,501 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:44,501 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T18:13:44,502 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T18:13:44,502 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T18:13:44,502 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. because compaction request was cancelled 2024-11-11T18:13:44,502 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:44,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-11T18:13:44,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:44,571 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:13:44,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:44,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:44,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:44,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:44,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:44,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:44,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a25957e25d394d2b97526fc2d4ed8efe is 50, key is test_row_0/A:col10/1731348823545/Put/seqid=0 2024-11-11T18:13:44,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741958_1134 (size=12301) 2024-11-11T18:13:44,628 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a25957e25d394d2b97526fc2d4ed8efe 2024-11-11T18:13:44,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/816b1b3e12924644a0c5b66ff9a8deea is 50, key is test_row_0/B:col10/1731348823545/Put/seqid=0 2024-11-11T18:13:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:44,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:44,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741959_1135 (size=12301) 2024-11-11T18:13:44,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348884694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348884695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348884695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348884696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348884701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348884804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348884814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:44,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:44,897 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/98b474e5a34c431989e96e139bc31247 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98b474e5a34c431989e96e139bc31247 2024-11-11T18:13:44,903 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into 98b474e5a34c431989e96e139bc31247(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:44,903 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:44,903 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348824441; duration=0sec 2024-11-11T18:13:44,904 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:44,904 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T18:13:45,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348885014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348885024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,056 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/816b1b3e12924644a0c5b66ff9a8deea 2024-11-11T18:13:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/9c459866189e479ba136c55fb085140c is 50, key is test_row_0/C:col10/1731348823545/Put/seqid=0 2024-11-11T18:13:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741960_1136 (size=12301) 2024-11-11T18:13:45,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348885325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348885334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,473 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=643 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/9c459866189e479ba136c55fb085140c 2024-11-11T18:13:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/a25957e25d394d2b97526fc2d4ed8efe as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a25957e25d394d2b97526fc2d4ed8efe 2024-11-11T18:13:45,484 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a25957e25d394d2b97526fc2d4ed8efe, entries=150, sequenceid=643, filesize=12.0 K 2024-11-11T18:13:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/816b1b3e12924644a0c5b66ff9a8deea as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/816b1b3e12924644a0c5b66ff9a8deea 2024-11-11T18:13:45,490 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/816b1b3e12924644a0c5b66ff9a8deea, entries=150, sequenceid=643, filesize=12.0 K 2024-11-11T18:13:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/9c459866189e479ba136c55fb085140c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9c459866189e479ba136c55fb085140c 2024-11-11T18:13:45,496 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9c459866189e479ba136c55fb085140c, entries=150, sequenceid=643, filesize=12.0 K 2024-11-11T18:13:45,497 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2330eb7332d026bb8d820d13c904c2d4 in 925ms, sequenceid=643, compaction requested=true 2024-11-11T18:13:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-11T18:13:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-11T18:13:45,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T18:13:45,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-11-11T18:13:45,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.7140 sec 2024-11-11T18:13:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:45,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T18:13:45,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:45,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:45,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:45,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c5be4d57c9d84ce89fb7a472e5e22887 is 50, key is test_row_0/A:col10/1731348824677/Put/seqid=0 2024-11-11T18:13:45,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741961_1137 (size=17181) 2024-11-11T18:13:45,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348885753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348885758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348885759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348885834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348885844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348885864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348885875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348885875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:45,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T18:13:45,935 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:45,936 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees 2024-11-11T18:13:45,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:45,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:45,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:45,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348886075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348886084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348886084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c5be4d57c9d84ce89fb7a472e5e22887 2024-11-11T18:13:46,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/539659d94d82446bae877d751cdaa0ca is 50, key is test_row_0/B:col10/1731348824677/Put/seqid=0 2024-11-11T18:13:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741962_1138 (size=12301) 2024-11-11T18:13:46,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/539659d94d82446bae877d751cdaa0ca 2024-11-11T18:13:46,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/17de9e42939e41a98ee43375c0cb8978 is 50, key is test_row_0/C:col10/1731348824677/Put/seqid=0 2024-11-11T18:13:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741963_1139 (size=12301) 2024-11-11T18:13:46,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/17de9e42939e41a98ee43375c0cb8978 2024-11-11T18:13:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/c5be4d57c9d84ce89fb7a472e5e22887 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c5be4d57c9d84ce89fb7a472e5e22887 2024-11-11T18:13:46,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c5be4d57c9d84ce89fb7a472e5e22887, entries=250, sequenceid=658, filesize=16.8 K 2024-11-11T18:13:46,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/539659d94d82446bae877d751cdaa0ca as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/539659d94d82446bae877d751cdaa0ca 2024-11-11T18:13:46,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/539659d94d82446bae877d751cdaa0ca, entries=150, sequenceid=658, filesize=12.0 K 2024-11-11T18:13:46,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/17de9e42939e41a98ee43375c0cb8978 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/17de9e42939e41a98ee43375c0cb8978 2024-11-11T18:13:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:46,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/17de9e42939e41a98ee43375c0cb8978, entries=150, sequenceid=658, filesize=12.0 K 2024-11-11T18:13:46,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2330eb7332d026bb8d820d13c904c2d4 in 538ms, sequenceid=658, compaction requested=true 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:46,259 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:46,259 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2330eb7332d026bb8d820d13c904c2d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:46,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:46,260 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43145 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:46,260 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/A is initiating minor compaction (all files) 2024-11-11T18:13:46,261 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/A in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,261 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98b474e5a34c431989e96e139bc31247, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a25957e25d394d2b97526fc2d4ed8efe, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c5be4d57c9d84ce89fb7a472e5e22887] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=42.1 K 2024-11-11T18:13:46,261 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:46,261 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b474e5a34c431989e96e139bc31247, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1731348823502 2024-11-11T18:13:46,261 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/B is initiating minor compaction (all files) 2024-11-11T18:13:46,262 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/B in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,262 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/9081f519fe614af9a1e46bccfe49ab55, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/816b1b3e12924644a0c5b66ff9a8deea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/539659d94d82446bae877d751cdaa0ca] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=37.4 K 2024-11-11T18:13:46,262 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a25957e25d394d2b97526fc2d4ed8efe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731348823540 2024-11-11T18:13:46,263 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9081f519fe614af9a1e46bccfe49ab55, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1731348823502 2024-11-11T18:13:46,263 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 816b1b3e12924644a0c5b66ff9a8deea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731348823540 2024-11-11T18:13:46,264 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 539659d94d82446bae877d751cdaa0ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731348824677 2024-11-11T18:13:46,265 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5be4d57c9d84ce89fb7a472e5e22887, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731348824677 2024-11-11T18:13:46,280 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#A#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:46,281 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b7f17ba9a0164316b05f2d35d07b9d72 is 50, key is test_row_0/A:col10/1731348824677/Put/seqid=0 2024-11-11T18:13:46,282 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#B#compaction#128 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:46,283 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3f66e6e4ae6f494eba0a9a02a69a3583 is 50, key is test_row_0/B:col10/1731348824677/Put/seqid=0 2024-11-11T18:13:46,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741965_1141 (size=13765) 2024-11-11T18:13:46,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741964_1140 (size=13765) 2024-11-11T18:13:46,310 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/3f66e6e4ae6f494eba0a9a02a69a3583 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3f66e6e4ae6f494eba0a9a02a69a3583 2024-11-11T18:13:46,315 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/b7f17ba9a0164316b05f2d35d07b9d72 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b7f17ba9a0164316b05f2d35d07b9d72 2024-11-11T18:13:46,318 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/B of 2330eb7332d026bb8d820d13c904c2d4 into 3f66e6e4ae6f494eba0a9a02a69a3583(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:46,319 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:46,319 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/B, priority=13, startTime=1731348826259; duration=0sec 2024-11-11T18:13:46,319 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:46,319 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:B 2024-11-11T18:13:46,319 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:13:46,325 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:13:46,325 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 2330eb7332d026bb8d820d13c904c2d4/C is initiating minor compaction (all files) 2024-11-11T18:13:46,325 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2330eb7332d026bb8d820d13c904c2d4/C in TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,325 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c9c83c2af9649aea40e9c13b4ca6767, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/8cd1e3bd1dff4b74997a28a8371cc6e6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9c459866189e479ba136c55fb085140c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/17de9e42939e41a98ee43375c0cb8978] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp, totalSize=49.3 K 2024-11-11T18:13:46,325 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c9c83c2af9649aea40e9c13b4ca6767, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731348822840 2024-11-11T18:13:46,326 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd1e3bd1dff4b74997a28a8371cc6e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=619, earliestPutTs=1731348823502 2024-11-11T18:13:46,327 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c459866189e479ba136c55fb085140c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=643, earliestPutTs=1731348823540 2024-11-11T18:13:46,327 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 17de9e42939e41a98ee43375c0cb8978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1731348824677 2024-11-11T18:13:46,330 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/A of 2330eb7332d026bb8d820d13c904c2d4 into b7f17ba9a0164316b05f2d35d07b9d72(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:46,330 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:46,330 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/A, priority=13, startTime=1731348826259; duration=0sec 2024-11-11T18:13:46,330 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:46,330 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:A 2024-11-11T18:13:46,340 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2330eb7332d026bb8d820d13c904c2d4#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:46,341 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/66bc9488beb4420192ca78dbc6969575 is 50, key is test_row_0/C:col10/1731348824677/Put/seqid=0 2024-11-11T18:13:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741966_1142 (size=13697) 2024-11-11T18:13:46,363 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/66bc9488beb4420192ca78dbc6969575 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/66bc9488beb4420192ca78dbc6969575 2024-11-11T18:13:46,374 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2330eb7332d026bb8d820d13c904c2d4/C of 2330eb7332d026bb8d820d13c904c2d4 into 66bc9488beb4420192ca78dbc6969575(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:46,374 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:46,374 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., storeName=2330eb7332d026bb8d820d13c904c2d4/C, priority=12, startTime=1731348826259; duration=0sec 2024-11-11T18:13:46,374 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:46,374 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2330eb7332d026bb8d820d13c904c2d4:C 2024-11-11T18:13:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:46,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T18:13:46,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:46,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:46,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:46,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:46,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:46,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:46,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4952416c332d4aa2bc59f970081508da is 50, key is test_row_0/A:col10/1731348825758/Put/seqid=0 2024-11-11T18:13:46,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:46,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741967_1143 (size=14741) 2024-11-11T18:13:46,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348886414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348886415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348886416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,491 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T18:13:46,491 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:46,492 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T18:13:46,492 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:46,494 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T18:13:46,494 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:46,497 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,497 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,497 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,497 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,497 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:46,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:46,498 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,498 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:46,500 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T18:13:46,500 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:46,500 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,500 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:46,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348886525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348886525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348886525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:46,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:46,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348886735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348886735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348886735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4952416c332d4aa2bc59f970081508da 2024-11-11T18:13:46,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/464afc004b7a4482a745f9888532bf1d is 50, key is test_row_0/B:col10/1731348825758/Put/seqid=0 2024-11-11T18:13:46,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741968_1144 (size=12301) 2024-11-11T18:13:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35864 deadline: 1731348886845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,846 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 2330eb7332d026bb8d820d13c904c2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:13:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731348886854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:46,854 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 2330eb7332d026bb8d820d13c904c2d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:13:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:46,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:46,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348887044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348887044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:47,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348887044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:47,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:47,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:47,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/464afc004b7a4482a745f9888532bf1d 2024-11-11T18:13:47,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/b11e1482dfe446ee997a0d3c6168d8b8 is 50, key is test_row_0/C:col10/1731348825758/Put/seqid=0 2024-11-11T18:13:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741969_1145 (size=12301) 2024-11-11T18:13:47,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:47,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:47,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:47,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35906 deadline: 1731348887554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35876 deadline: 1731348887554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35894 deadline: 1731348887554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:13:47,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:47,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:47,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/b11e1482dfe446ee997a0d3c6168d8b8 2024-11-11T18:13:47,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/4952416c332d4aa2bc59f970081508da as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4952416c332d4aa2bc59f970081508da 2024-11-11T18:13:47,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4952416c332d4aa2bc59f970081508da, entries=200, sequenceid=680, filesize=14.4 K 2024-11-11T18:13:47,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/464afc004b7a4482a745f9888532bf1d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/464afc004b7a4482a745f9888532bf1d 2024-11-11T18:13:47,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/464afc004b7a4482a745f9888532bf1d, entries=150, sequenceid=680, filesize=12.0 K 2024-11-11T18:13:47,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/b11e1482dfe446ee997a0d3c6168d8b8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/b11e1482dfe446ee997a0d3c6168d8b8 2024-11-11T18:13:47,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/b11e1482dfe446ee997a0d3c6168d8b8, entries=150, sequenceid=680, filesize=12.0 K 2024-11-11T18:13:47,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2330eb7332d026bb8d820d13c904c2d4 in 1265ms, sequenceid=680, compaction requested=false 2024-11-11T18:13:47,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:47,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-11T18:13:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:47,783 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T18:13:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:47,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:47,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/9690b2f5d2aa477280fdccf43498cb33 is 50, key is test_row_0/A:col10/1731348826413/Put/seqid=0 2024-11-11T18:13:47,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741970_1146 (size=12301) 2024-11-11T18:13:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:48,192 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/9690b2f5d2aa477280fdccf43498cb33 2024-11-11T18:13:48,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/01265ff7c23646058d1d41aa002b728c is 50, key is test_row_0/B:col10/1731348826413/Put/seqid=0 2024-11-11T18:13:48,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741971_1147 (size=12301) 2024-11-11T18:13:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. as already flushing 2024-11-11T18:13:48,566 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T18:13:48,566 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T18:13:48,566 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T18:13:48,566 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:48,566 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,566 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:48,566 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:48,566 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,566 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,566 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,566 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,566 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:48,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:48,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:48,604 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/01265ff7c23646058d1d41aa002b728c 2024-11-11T18:13:48,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/2034576c09494a06819531b2796b9759 is 50, key is test_row_0/C:col10/1731348826413/Put/seqid=0 2024-11-11T18:13:48,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741972_1148 (size=12301) 2024-11-11T18:13:48,809 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T18:13:48,866 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T18:13:48,866 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:48,866 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,866 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:48,876 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T18:13:48,876 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:13:48,876 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,876 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:48,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:49,016 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=693 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/2034576c09494a06819531b2796b9759 2024-11-11T18:13:49,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/9690b2f5d2aa477280fdccf43498cb33 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/9690b2f5d2aa477280fdccf43498cb33 2024-11-11T18:13:49,025 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/9690b2f5d2aa477280fdccf43498cb33, entries=150, sequenceid=693, filesize=12.0 K 2024-11-11T18:13:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/01265ff7c23646058d1d41aa002b728c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/01265ff7c23646058d1d41aa002b728c 2024-11-11T18:13:49,030 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/01265ff7c23646058d1d41aa002b728c, entries=150, sequenceid=693, filesize=12.0 K 2024-11-11T18:13:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/2034576c09494a06819531b2796b9759 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2034576c09494a06819531b2796b9759 2024-11-11T18:13:49,034 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2034576c09494a06819531b2796b9759, entries=150, sequenceid=693, filesize=12.0 K 2024-11-11T18:13:49,035 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=33.54 KB/34350 for 2330eb7332d026bb8d820d13c904c2d4 in 1252ms, sequenceid=693, compaction requested=true 2024-11-11T18:13:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for 2330eb7332d026bb8d820d13c904c2d4: 2024-11-11T18:13:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-11T18:13:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-11T18:13:49,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T18:13:49,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0970 sec 2024-11-11T18:13:49,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees in 3.1030 sec 2024-11-11T18:13:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T18:13:50,105 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2374 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2422 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T18:13:50,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1112 2024-11-11T18:13:50,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3332 rows 2024-11-11T18:13:50,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1112 2024-11-11T18:13:50,106 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3333 rows 2024-11-11T18:13:50,106 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:13:50,106 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:13:50,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:50,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:50,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:13:50,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T18:13:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T18:13:50,117 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348830117"}]},"ts":"1731348830117"} 2024-11-11T18:13:50,119 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T18:13:50,119 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T18:13:50,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:13:50,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, UNASSIGN}] 2024-11-11T18:13:50,126 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, UNASSIGN 2024-11-11T18:13:50,127 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2330eb7332d026bb8d820d13c904c2d4, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:50,129 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41925 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=90f34c5bee6a,42957,1731348800579, table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T18:13:50,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, UNASSIGN because future has completed 2024-11-11T18:13:50,130 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:13:50,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:13:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T18:13:50,287 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:50,287 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 2330eb7332d026bb8d820d13c904c2d4, disabling compactions & flushes 2024-11-11T18:13:50,288 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. after waiting 0 ms 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:50,288 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2902): Flushing 2330eb7332d026bb8d820d13c904c2d4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=A 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=B 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:50,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2330eb7332d026bb8d820d13c904c2d4, store=C 2024-11-11T18:13:50,289 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:50,293 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/eaff6c68b8d741bdaa48950102369dff is 50, key is test_row_0/A:col10/1731348828565/Put/seqid=0 2024-11-11T18:13:50,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741973_1149 (size=12301) 2024-11-11T18:13:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T18:13:50,697 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/eaff6c68b8d741bdaa48950102369dff 2024-11-11T18:13:50,704 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/db63dc40946d40c5bda5043fd3b52e85 is 50, key is test_row_0/B:col10/1731348828565/Put/seqid=0 2024-11-11T18:13:50,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741974_1150 (size=12301) 2024-11-11T18:13:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T18:13:51,109 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/db63dc40946d40c5bda5043fd3b52e85 2024-11-11T18:13:51,117 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/cc21447cf00d4a91a4dc28c4993d3949 is 50, key is test_row_0/C:col10/1731348828565/Put/seqid=0 2024-11-11T18:13:51,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741975_1151 (size=12301) 2024-11-11T18:13:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T18:13:51,522 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=701 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/cc21447cf00d4a91a4dc28c4993d3949 2024-11-11T18:13:51,527 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/A/eaff6c68b8d741bdaa48950102369dff as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/eaff6c68b8d741bdaa48950102369dff 2024-11-11T18:13:51,531 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/eaff6c68b8d741bdaa48950102369dff, entries=150, sequenceid=701, filesize=12.0 K 2024-11-11T18:13:51,532 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/B/db63dc40946d40c5bda5043fd3b52e85 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/db63dc40946d40c5bda5043fd3b52e85 2024-11-11T18:13:51,536 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/db63dc40946d40c5bda5043fd3b52e85, entries=150, sequenceid=701, filesize=12.0 K 2024-11-11T18:13:51,537 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/.tmp/C/cc21447cf00d4a91a4dc28c4993d3949 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cc21447cf00d4a91a4dc28c4993d3949 2024-11-11T18:13:51,541 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cc21447cf00d4a91a4dc28c4993d3949, entries=150, sequenceid=701, filesize=12.0 K 2024-11-11T18:13:51,542 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2330eb7332d026bb8d820d13c904c2d4 in 1254ms, sequenceid=701, compaction requested=true 2024-11-11T18:13:51,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bb30798e5a54496b96fbf03694edf982, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/580643fe3af44e3289a7014f71465464, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5e8c8c1107ea4a319b693b1360d99e58, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/3829e36f949c4d74b924efca5028f2cd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/987934b65d414460a9216da30fee7e73, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4549acdfb9da4093a8c14f2628d45869, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5130019453854474b5acc4f9dcf78c2c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f3689a2f879a406b87826440d95262b2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c810a7382bea4eb8a503eebba77dd727, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/dc897c7b5f104c3fb63f34d0b3c5680c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6e8787d172ee42089aca40653adba2bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a5600f058f6d4647b45d87c5ba6cab36, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/562945a4df644d5d98668f0f6cf88b12, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bd0517d0cbb248ac8704a9896ba32a1d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/32a6d945fb974fedb7fa96c2c1b758d3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5fefcd432bed4135acce288aa8e9d973, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/7c24d7ede1dc4b7e898f5f7aca115bde, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b9193c478b6b4517bbc382e916633b3b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/da8cceff1b9f46df8f377d3773dc6b04, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/ce90803c397e48158e3b0c95fbb98a7e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b5e562fafa9a44278c9113dbefc49973, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/97ab27618c5d4eff8943d669753d769e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/fbbf463a53224a619cb6ca08914f06ea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/243404e4875e4dfb9a86ef89418e4af7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5967604977d345a5a5dffb07c8ea51ba, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/60ab294ed3114e409ef0228e85e50f50, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c0eb0729618d4c31ab673b8bc8c77112, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bcf208fda9534f7cb257d1fc5d85d241, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5a68f5cfc6d640328c01a280777438a6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c822fa7073ef4065b8c69fb2da9fce9f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f43ea31778ea45dcb1a5a826febd88d4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a0f4c367ff664b6abea8e25bd26abdac, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4ffa37ef47884dd0b13526602a1c5a63, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b6b9a2199d7c4c5e94e81597af675637, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/68dc44b80486461493ab65df790eeb1e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6c212eeefe8f4b538019be6a5897212e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/d4cad6206f2e43cd983b189fb0d34239, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98778a1497dc43cfb45ebbf0e6d4d633, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/e1ba2cd4ce6842959665b117ec5fd63e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98b474e5a34c431989e96e139bc31247, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a25957e25d394d2b97526fc2d4ed8efe, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c5be4d57c9d84ce89fb7a472e5e22887] to archive 2024-11-11T18:13:51,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:13:51,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bb30798e5a54496b96fbf03694edf982 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bb30798e5a54496b96fbf03694edf982 2024-11-11T18:13:51,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/580643fe3af44e3289a7014f71465464 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/580643fe3af44e3289a7014f71465464 2024-11-11T18:13:51,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5e8c8c1107ea4a319b693b1360d99e58 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5e8c8c1107ea4a319b693b1360d99e58 2024-11-11T18:13:51,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/3829e36f949c4d74b924efca5028f2cd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/3829e36f949c4d74b924efca5028f2cd 2024-11-11T18:13:51,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/987934b65d414460a9216da30fee7e73 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/987934b65d414460a9216da30fee7e73 2024-11-11T18:13:51,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4549acdfb9da4093a8c14f2628d45869 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4549acdfb9da4093a8c14f2628d45869 2024-11-11T18:13:51,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5130019453854474b5acc4f9dcf78c2c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5130019453854474b5acc4f9dcf78c2c 2024-11-11T18:13:51,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f3689a2f879a406b87826440d95262b2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f3689a2f879a406b87826440d95262b2 2024-11-11T18:13:51,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c810a7382bea4eb8a503eebba77dd727 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c810a7382bea4eb8a503eebba77dd727 2024-11-11T18:13:51,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/dc897c7b5f104c3fb63f34d0b3c5680c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/dc897c7b5f104c3fb63f34d0b3c5680c 2024-11-11T18:13:51,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6e8787d172ee42089aca40653adba2bb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6e8787d172ee42089aca40653adba2bb 2024-11-11T18:13:51,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a5600f058f6d4647b45d87c5ba6cab36 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a5600f058f6d4647b45d87c5ba6cab36 2024-11-11T18:13:51,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/562945a4df644d5d98668f0f6cf88b12 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/562945a4df644d5d98668f0f6cf88b12 2024-11-11T18:13:51,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bd0517d0cbb248ac8704a9896ba32a1d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bd0517d0cbb248ac8704a9896ba32a1d 2024-11-11T18:13:51,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/32a6d945fb974fedb7fa96c2c1b758d3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/32a6d945fb974fedb7fa96c2c1b758d3 2024-11-11T18:13:51,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5fefcd432bed4135acce288aa8e9d973 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5fefcd432bed4135acce288aa8e9d973 2024-11-11T18:13:51,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/7c24d7ede1dc4b7e898f5f7aca115bde to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/7c24d7ede1dc4b7e898f5f7aca115bde 2024-11-11T18:13:51,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b9193c478b6b4517bbc382e916633b3b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b9193c478b6b4517bbc382e916633b3b 2024-11-11T18:13:51,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/da8cceff1b9f46df8f377d3773dc6b04 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/da8cceff1b9f46df8f377d3773dc6b04 2024-11-11T18:13:51,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5b7ac5fe4c7f45fdb4982ac008cbcdc6 2024-11-11T18:13:51,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/ce90803c397e48158e3b0c95fbb98a7e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/ce90803c397e48158e3b0c95fbb98a7e 2024-11-11T18:13:51,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b5e562fafa9a44278c9113dbefc49973 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b5e562fafa9a44278c9113dbefc49973 2024-11-11T18:13:51,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/97ab27618c5d4eff8943d669753d769e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/97ab27618c5d4eff8943d669753d769e 2024-11-11T18:13:51,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/fbbf463a53224a619cb6ca08914f06ea to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/fbbf463a53224a619cb6ca08914f06ea 2024-11-11T18:13:51,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/243404e4875e4dfb9a86ef89418e4af7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/243404e4875e4dfb9a86ef89418e4af7 2024-11-11T18:13:51,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5967604977d345a5a5dffb07c8ea51ba to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5967604977d345a5a5dffb07c8ea51ba 2024-11-11T18:13:51,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/60ab294ed3114e409ef0228e85e50f50 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/60ab294ed3114e409ef0228e85e50f50 2024-11-11T18:13:51,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c0eb0729618d4c31ab673b8bc8c77112 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c0eb0729618d4c31ab673b8bc8c77112 2024-11-11T18:13:51,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bcf208fda9534f7cb257d1fc5d85d241 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/bcf208fda9534f7cb257d1fc5d85d241 2024-11-11T18:13:51,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5a68f5cfc6d640328c01a280777438a6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/5a68f5cfc6d640328c01a280777438a6 2024-11-11T18:13:51,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c822fa7073ef4065b8c69fb2da9fce9f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c822fa7073ef4065b8c69fb2da9fce9f 2024-11-11T18:13:51,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f43ea31778ea45dcb1a5a826febd88d4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/f43ea31778ea45dcb1a5a826febd88d4 2024-11-11T18:13:51,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a0f4c367ff664b6abea8e25bd26abdac to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a0f4c367ff664b6abea8e25bd26abdac 2024-11-11T18:13:51,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4ffa37ef47884dd0b13526602a1c5a63 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4ffa37ef47884dd0b13526602a1c5a63 2024-11-11T18:13:51,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b6b9a2199d7c4c5e94e81597af675637 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b6b9a2199d7c4c5e94e81597af675637 2024-11-11T18:13:51,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/68dc44b80486461493ab65df790eeb1e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/68dc44b80486461493ab65df790eeb1e 2024-11-11T18:13:51,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6c212eeefe8f4b538019be6a5897212e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/6c212eeefe8f4b538019be6a5897212e 2024-11-11T18:13:51,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/d4cad6206f2e43cd983b189fb0d34239 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/d4cad6206f2e43cd983b189fb0d34239 2024-11-11T18:13:51,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98778a1497dc43cfb45ebbf0e6d4d633 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98778a1497dc43cfb45ebbf0e6d4d633 2024-11-11T18:13:51,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/e1ba2cd4ce6842959665b117ec5fd63e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/e1ba2cd4ce6842959665b117ec5fd63e 2024-11-11T18:13:51,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98b474e5a34c431989e96e139bc31247 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/98b474e5a34c431989e96e139bc31247 2024-11-11T18:13:51,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a25957e25d394d2b97526fc2d4ed8efe to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/a25957e25d394d2b97526fc2d4ed8efe 2024-11-11T18:13:51,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c5be4d57c9d84ce89fb7a472e5e22887 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/c5be4d57c9d84ce89fb7a472e5e22887 2024-11-11T18:13:51,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/926896a3d5aa444cad5565807480e73a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ba6ee1ab4d5c479890508e4b4d389adb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b7c0b18b86a549758e7079ce5bfdfb24, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3bad7c60614d4658bad04ba298359405, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/216a38db5b72413dbaf5ea5c962d39cc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b467787931df40539dabb1be35f42306, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/10c4ff83f6264e3988fcb285c5185401, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f2fce18869de4739afc01fef0d89da5f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/e5884eb9d4114752a7f54d1453ab785c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/97c2ec3977f04f69a193130505c232bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1d81ed5ebde4bf6a6046d955965891d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ae2a82f0206c400d82fa15af2c3bd69a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/948db61347174b4c9ed1556662944d70, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d14f778f186742b897e2d23df5fb7a05, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8d73efa896b2447ca57964a003fd845c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/45f0f41682cb4269aa17884226f5bbd3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d40e4abdfc6a46719dba43e67c501946, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/91c620d174394d23afda3a08f959b999, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/eaa0c1cdcc594738a2a9d4d74250bcc5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f1a0b2ee47f74c8597d4923de0a312b4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/bbace6698b81411e87a7b3836d8a1c18, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ad2a35e4ff204b619a3b6cfa21fb4b16, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a7e0c93c1dd047d4b8c0d556f81af5d7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/4450706b157140fc865bda84ac5c9aa9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/6e430caef278409cad27378e2b0ee656, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2ae2f2db92ce4ef78e69b61308cd1eb6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1523e22d0254bfa968dee7b8f59a94c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d71b197ef04e40eda75daacd9b590b8c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/13bbbcf31fb544e2a4f2f6d9e435fa96, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2e2efbcee0d24dcab285e1e219f54439, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a45da1308b3c4453adf02e5fe66cc1d1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/707a665158c442d399aa9891640c4072, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/7470b38d9c9b4678a85136152efbca97, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/135baafef2f14a4a80383ed6ceae3a2c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3ceeb4c364774581a5aa14b08952dccd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8ae3c3cf1f2743ba94958f2da251e5d2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3987e8fbfd974d8fbeb151253548e01a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/90ee7a08c09642b18dc61a0bcbfcb3ee, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/11819673050d4f099dc85a5afd1a62a1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/9081f519fe614af9a1e46bccfe49ab55, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f098a694b45c4446bfa36225c2cb14c5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/816b1b3e12924644a0c5b66ff9a8deea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/539659d94d82446bae877d751cdaa0ca] to archive 2024-11-11T18:13:51,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:13:51,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/926896a3d5aa444cad5565807480e73a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/926896a3d5aa444cad5565807480e73a 2024-11-11T18:13:51,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ba6ee1ab4d5c479890508e4b4d389adb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ba6ee1ab4d5c479890508e4b4d389adb 2024-11-11T18:13:51,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b7c0b18b86a549758e7079ce5bfdfb24 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b7c0b18b86a549758e7079ce5bfdfb24 2024-11-11T18:13:51,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3bad7c60614d4658bad04ba298359405 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3bad7c60614d4658bad04ba298359405 2024-11-11T18:13:51,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/216a38db5b72413dbaf5ea5c962d39cc to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/216a38db5b72413dbaf5ea5c962d39cc 2024-11-11T18:13:51,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b467787931df40539dabb1be35f42306 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/b467787931df40539dabb1be35f42306 2024-11-11T18:13:51,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/10c4ff83f6264e3988fcb285c5185401 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/10c4ff83f6264e3988fcb285c5185401 2024-11-11T18:13:51,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f2fce18869de4739afc01fef0d89da5f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f2fce18869de4739afc01fef0d89da5f 2024-11-11T18:13:51,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/e5884eb9d4114752a7f54d1453ab785c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/e5884eb9d4114752a7f54d1453ab785c 2024-11-11T18:13:51,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/97c2ec3977f04f69a193130505c232bb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/97c2ec3977f04f69a193130505c232bb 2024-11-11T18:13:51,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1d81ed5ebde4bf6a6046d955965891d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1d81ed5ebde4bf6a6046d955965891d 2024-11-11T18:13:51,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ae2a82f0206c400d82fa15af2c3bd69a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ae2a82f0206c400d82fa15af2c3bd69a 2024-11-11T18:13:51,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/948db61347174b4c9ed1556662944d70 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/948db61347174b4c9ed1556662944d70 2024-11-11T18:13:51,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d14f778f186742b897e2d23df5fb7a05 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d14f778f186742b897e2d23df5fb7a05 2024-11-11T18:13:51,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8d73efa896b2447ca57964a003fd845c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8d73efa896b2447ca57964a003fd845c 2024-11-11T18:13:51,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/45f0f41682cb4269aa17884226f5bbd3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/45f0f41682cb4269aa17884226f5bbd3 2024-11-11T18:13:51,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d40e4abdfc6a46719dba43e67c501946 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d40e4abdfc6a46719dba43e67c501946 2024-11-11T18:13:51,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/91c620d174394d23afda3a08f959b999 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/91c620d174394d23afda3a08f959b999 2024-11-11T18:13:51,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/eaa0c1cdcc594738a2a9d4d74250bcc5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/eaa0c1cdcc594738a2a9d4d74250bcc5 2024-11-11T18:13:51,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f1a0b2ee47f74c8597d4923de0a312b4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f1a0b2ee47f74c8597d4923de0a312b4 2024-11-11T18:13:51,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/bbace6698b81411e87a7b3836d8a1c18 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/bbace6698b81411e87a7b3836d8a1c18 2024-11-11T18:13:51,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ad2a35e4ff204b619a3b6cfa21fb4b16 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/ad2a35e4ff204b619a3b6cfa21fb4b16 2024-11-11T18:13:51,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a7e0c93c1dd047d4b8c0d556f81af5d7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a7e0c93c1dd047d4b8c0d556f81af5d7 2024-11-11T18:13:51,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/4450706b157140fc865bda84ac5c9aa9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/4450706b157140fc865bda84ac5c9aa9 2024-11-11T18:13:51,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/6e430caef278409cad27378e2b0ee656 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/6e430caef278409cad27378e2b0ee656 2024-11-11T18:13:51,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2ae2f2db92ce4ef78e69b61308cd1eb6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2ae2f2db92ce4ef78e69b61308cd1eb6 2024-11-11T18:13:51,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1523e22d0254bfa968dee7b8f59a94c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a1523e22d0254bfa968dee7b8f59a94c 2024-11-11T18:13:51,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d71b197ef04e40eda75daacd9b590b8c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/d71b197ef04e40eda75daacd9b590b8c 2024-11-11T18:13:51,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/13bbbcf31fb544e2a4f2f6d9e435fa96 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/13bbbcf31fb544e2a4f2f6d9e435fa96 2024-11-11T18:13:51,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2e2efbcee0d24dcab285e1e219f54439 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/2e2efbcee0d24dcab285e1e219f54439 2024-11-11T18:13:51,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a45da1308b3c4453adf02e5fe66cc1d1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/a45da1308b3c4453adf02e5fe66cc1d1 2024-11-11T18:13:51,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/707a665158c442d399aa9891640c4072 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/707a665158c442d399aa9891640c4072 2024-11-11T18:13:51,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/7470b38d9c9b4678a85136152efbca97 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/7470b38d9c9b4678a85136152efbca97 2024-11-11T18:13:51,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/135baafef2f14a4a80383ed6ceae3a2c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/135baafef2f14a4a80383ed6ceae3a2c 2024-11-11T18:13:51,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3ceeb4c364774581a5aa14b08952dccd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3ceeb4c364774581a5aa14b08952dccd 2024-11-11T18:13:51,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8ae3c3cf1f2743ba94958f2da251e5d2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/8ae3c3cf1f2743ba94958f2da251e5d2 2024-11-11T18:13:51,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3987e8fbfd974d8fbeb151253548e01a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3987e8fbfd974d8fbeb151253548e01a 2024-11-11T18:13:51,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/90ee7a08c09642b18dc61a0bcbfcb3ee to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/90ee7a08c09642b18dc61a0bcbfcb3ee 2024-11-11T18:13:51,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/11819673050d4f099dc85a5afd1a62a1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/11819673050d4f099dc85a5afd1a62a1 2024-11-11T18:13:51,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/9081f519fe614af9a1e46bccfe49ab55 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/9081f519fe614af9a1e46bccfe49ab55 2024-11-11T18:13:51,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f098a694b45c4446bfa36225c2cb14c5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/f098a694b45c4446bfa36225c2cb14c5 2024-11-11T18:13:51,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/816b1b3e12924644a0c5b66ff9a8deea to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/816b1b3e12924644a0c5b66ff9a8deea 2024-11-11T18:13:51,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/539659d94d82446bae877d751cdaa0ca to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/539659d94d82446bae877d751cdaa0ca 2024-11-11T18:13:51,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/4986c5f32c4f4c2c93e3014db8c3de5d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/01a00bd13c924782893c4bf6921850df, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9aa79b0d1560402fbc4f0baf609a2bdd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/956a55f1cd2447bfbc8a51ced6e72dad, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1b4e8899da1c469294a36428e2c77a70, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/75de7339b5894245bbc51e24aeeec814, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/597b11e53c1e4e97971e999f090c49dc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/6a60fea6025e4a28989141935e3e9ab9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/92675d5d4e8f442dbd371b79fe3b6189, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/0ccec2a2cd7f4c289dc0d39d628d1368, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/a472dbf39e2542fca747798ba5dd2a40, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2cfdb5bc766c4d4fa5371deff2fbb210, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d576e129a07e40759d409c779e99f380, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/67ff1dc8096f4c4d9258fec97bc2933d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d981279fe0dc42988849f2822daeec9f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/069bb088dc48439ab73fd29189dcb768, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7d870242bf4490ab14f042d537b5ac1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/79740674281546cead3fd9247fd9638f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7f7b71d9d81411ca4787b9cb3bd5bbd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/25d6a33dadc340688f6ded0ec60ff666, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/38666a9dd1d447048e051dd099d6706c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1cf24690a1ba4c839ef2ee9f57625c9b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/fb50520ed55c46ecb6854b6be5725ff7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/954507c092394862a95e131fd4cad6c7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/7b0fb2a50a484a8d8cec4144586d76b6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ce3b6785bb2141ccac7fd7d3472911c1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d635e0faa27949f6ab9a2f5b33094a0c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c1fe39f45af4690b1cc53e89a1c4ef3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d96102086d4c4bec95f5cdaa0172ba5b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d97bca450c9e4464b82570837ab4ad3b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cad9ed4d32b5426eb74d61cbfb66a690, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/24422fef92764f58b3c6724ee03b645d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ae8181a614224cb1b5fb8041437c7a53, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/52ad74d7003948ce9040401841dad75a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/f74259b2b0944fd0aeda96363cfc1fe0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/584d6c045bc94a6298c19b1cd0071f3d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c9c83c2af9649aea40e9c13b4ca6767, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/15a4c208c43043c29672986254b453c1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/8cd1e3bd1dff4b74997a28a8371cc6e6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9c459866189e479ba136c55fb085140c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/17de9e42939e41a98ee43375c0cb8978] to archive 2024-11-11T18:13:51,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:13:51,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/4986c5f32c4f4c2c93e3014db8c3de5d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/4986c5f32c4f4c2c93e3014db8c3de5d 2024-11-11T18:13:51,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/01a00bd13c924782893c4bf6921850df to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/01a00bd13c924782893c4bf6921850df 2024-11-11T18:13:51,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9aa79b0d1560402fbc4f0baf609a2bdd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9aa79b0d1560402fbc4f0baf609a2bdd 2024-11-11T18:13:51,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/956a55f1cd2447bfbc8a51ced6e72dad to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/956a55f1cd2447bfbc8a51ced6e72dad 2024-11-11T18:13:51,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1b4e8899da1c469294a36428e2c77a70 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1b4e8899da1c469294a36428e2c77a70 2024-11-11T18:13:51,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/75de7339b5894245bbc51e24aeeec814 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/75de7339b5894245bbc51e24aeeec814 2024-11-11T18:13:51,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/597b11e53c1e4e97971e999f090c49dc to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/597b11e53c1e4e97971e999f090c49dc 2024-11-11T18:13:51,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/6a60fea6025e4a28989141935e3e9ab9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/6a60fea6025e4a28989141935e3e9ab9 2024-11-11T18:13:51,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/92675d5d4e8f442dbd371b79fe3b6189 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/92675d5d4e8f442dbd371b79fe3b6189 2024-11-11T18:13:51,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/0ccec2a2cd7f4c289dc0d39d628d1368 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/0ccec2a2cd7f4c289dc0d39d628d1368 2024-11-11T18:13:51,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/a472dbf39e2542fca747798ba5dd2a40 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/a472dbf39e2542fca747798ba5dd2a40 2024-11-11T18:13:51,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2cfdb5bc766c4d4fa5371deff2fbb210 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2cfdb5bc766c4d4fa5371deff2fbb210 2024-11-11T18:13:51,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d576e129a07e40759d409c779e99f380 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d576e129a07e40759d409c779e99f380 2024-11-11T18:13:51,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/67ff1dc8096f4c4d9258fec97bc2933d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/67ff1dc8096f4c4d9258fec97bc2933d 2024-11-11T18:13:51,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d981279fe0dc42988849f2822daeec9f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d981279fe0dc42988849f2822daeec9f 2024-11-11T18:13:51,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/069bb088dc48439ab73fd29189dcb768 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/069bb088dc48439ab73fd29189dcb768 2024-11-11T18:13:51,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7d870242bf4490ab14f042d537b5ac1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7d870242bf4490ab14f042d537b5ac1 2024-11-11T18:13:51,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/79740674281546cead3fd9247fd9638f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/79740674281546cead3fd9247fd9638f 2024-11-11T18:13:51,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7f7b71d9d81411ca4787b9cb3bd5bbd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/c7f7b71d9d81411ca4787b9cb3bd5bbd 2024-11-11T18:13:51,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/25d6a33dadc340688f6ded0ec60ff666 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/25d6a33dadc340688f6ded0ec60ff666 2024-11-11T18:13:51,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/38666a9dd1d447048e051dd099d6706c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/38666a9dd1d447048e051dd099d6706c 2024-11-11T18:13:51,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1cf24690a1ba4c839ef2ee9f57625c9b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1cf24690a1ba4c839ef2ee9f57625c9b 2024-11-11T18:13:51,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/fb50520ed55c46ecb6854b6be5725ff7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/fb50520ed55c46ecb6854b6be5725ff7 2024-11-11T18:13:51,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/954507c092394862a95e131fd4cad6c7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/954507c092394862a95e131fd4cad6c7 2024-11-11T18:13:51,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/7b0fb2a50a484a8d8cec4144586d76b6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/7b0fb2a50a484a8d8cec4144586d76b6 2024-11-11T18:13:51,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ce3b6785bb2141ccac7fd7d3472911c1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ce3b6785bb2141ccac7fd7d3472911c1 2024-11-11T18:13:51,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d635e0faa27949f6ab9a2f5b33094a0c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d635e0faa27949f6ab9a2f5b33094a0c 2024-11-11T18:13:51,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c1fe39f45af4690b1cc53e89a1c4ef3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c1fe39f45af4690b1cc53e89a1c4ef3 2024-11-11T18:13:51,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d96102086d4c4bec95f5cdaa0172ba5b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d96102086d4c4bec95f5cdaa0172ba5b 2024-11-11T18:13:51,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d97bca450c9e4464b82570837ab4ad3b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/d97bca450c9e4464b82570837ab4ad3b 2024-11-11T18:13:51,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cad9ed4d32b5426eb74d61cbfb66a690 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cad9ed4d32b5426eb74d61cbfb66a690 2024-11-11T18:13:51,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/24422fef92764f58b3c6724ee03b645d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/24422fef92764f58b3c6724ee03b645d 2024-11-11T18:13:51,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ae8181a614224cb1b5fb8041437c7a53 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/ae8181a614224cb1b5fb8041437c7a53 2024-11-11T18:13:51,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/52ad74d7003948ce9040401841dad75a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/52ad74d7003948ce9040401841dad75a 2024-11-11T18:13:51,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/f74259b2b0944fd0aeda96363cfc1fe0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/f74259b2b0944fd0aeda96363cfc1fe0 2024-11-11T18:13:51,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/584d6c045bc94a6298c19b1cd0071f3d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/584d6c045bc94a6298c19b1cd0071f3d 2024-11-11T18:13:51,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c9c83c2af9649aea40e9c13b4ca6767 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/1c9c83c2af9649aea40e9c13b4ca6767 2024-11-11T18:13:51,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/15a4c208c43043c29672986254b453c1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/15a4c208c43043c29672986254b453c1 2024-11-11T18:13:51,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/8cd1e3bd1dff4b74997a28a8371cc6e6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/8cd1e3bd1dff4b74997a28a8371cc6e6 2024-11-11T18:13:51,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9c459866189e479ba136c55fb085140c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/9c459866189e479ba136c55fb085140c 2024-11-11T18:13:51,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/17de9e42939e41a98ee43375c0cb8978 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/17de9e42939e41a98ee43375c0cb8978 2024-11-11T18:13:51,713 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/recovered.edits/704.seqid, newMaxSeqId=704, maxSeqId=1 2024-11-11T18:13:51,714 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. 2024-11-11T18:13:51,714 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 2330eb7332d026bb8d820d13c904c2d4: Waiting for close lock at 1731348830288Running coprocessor pre-close hooks at 1731348830288Disabling compacts and flushes for region at 1731348830288Disabling writes for close at 1731348830288Obtaining lock to block concurrent updates at 1731348830288Preparing flush snapshotting stores in 2330eb7332d026bb8d820d13c904c2d4 at 1731348830288Finished memstore snapshotting TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731348830289 (+1 ms)Flushing stores of TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4. at 1731348830289Flushing 2330eb7332d026bb8d820d13c904c2d4/A: creating writer at 1731348830289Flushing 2330eb7332d026bb8d820d13c904c2d4/A: appending metadata at 1731348830292 (+3 ms)Flushing 2330eb7332d026bb8d820d13c904c2d4/A: closing flushed file at 1731348830292Flushing 2330eb7332d026bb8d820d13c904c2d4/B: creating writer at 1731348830701 (+409 ms)Flushing 2330eb7332d026bb8d820d13c904c2d4/B: appending metadata at 1731348830704 (+3 ms)Flushing 2330eb7332d026bb8d820d13c904c2d4/B: closing flushed file at 1731348830704Flushing 2330eb7332d026bb8d820d13c904c2d4/C: creating writer at 1731348831114 (+410 ms)Flushing 2330eb7332d026bb8d820d13c904c2d4/C: appending metadata at 1731348831117 (+3 ms)Flushing 2330eb7332d026bb8d820d13c904c2d4/C: closing flushed file at 1731348831117Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@588a1b63: reopening flushed file at 1731348831526 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3739eb91: reopening flushed file at 1731348831531 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d2031f2: reopening flushed file at 1731348831536 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2330eb7332d026bb8d820d13c904c2d4 in 1254ms, sequenceid=701, compaction requested=true at 1731348831542 (+6 ms)Writing region close event to WAL at 1731348831710 (+168 ms)Running coprocessor post-close hooks at 1731348831714 (+4 ms)Closed at 1731348831714 2024-11-11T18:13:51,717 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:51,717 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=2330eb7332d026bb8d820d13c904c2d4, regionState=CLOSED 2024-11-11T18:13:51,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:13:51,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T18:13:51,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 2330eb7332d026bb8d820d13c904c2d4, server=90f34c5bee6a,42957,1731348800579 in 1.5900 sec 2024-11-11T18:13:51,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T18:13:51,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2330eb7332d026bb8d820d13c904c2d4, UNASSIGN in 1.5970 sec 2024-11-11T18:13:51,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T18:13:51,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6030 sec 2024-11-11T18:13:51,727 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348831727"}]},"ts":"1731348831727"} 2024-11-11T18:13:51,729 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T18:13:51,729 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T18:13:51,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6170 sec 2024-11-11T18:13:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T18:13:52,265 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:52,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T18:13:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,274 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-11T18:13:52,275 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=37, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,280 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:52,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/recovered.edits] 2024-11-11T18:13:52,287 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4952416c332d4aa2bc59f970081508da to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/4952416c332d4aa2bc59f970081508da 2024-11-11T18:13:52,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/9690b2f5d2aa477280fdccf43498cb33 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/9690b2f5d2aa477280fdccf43498cb33 2024-11-11T18:13:52,289 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b7f17ba9a0164316b05f2d35d07b9d72 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/b7f17ba9a0164316b05f2d35d07b9d72 2024-11-11T18:13:52,290 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/eaff6c68b8d741bdaa48950102369dff to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/A/eaff6c68b8d741bdaa48950102369dff 2024-11-11T18:13:52,293 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/01265ff7c23646058d1d41aa002b728c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/01265ff7c23646058d1d41aa002b728c 2024-11-11T18:13:52,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3f66e6e4ae6f494eba0a9a02a69a3583 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/3f66e6e4ae6f494eba0a9a02a69a3583 2024-11-11T18:13:52,295 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/464afc004b7a4482a745f9888532bf1d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/464afc004b7a4482a745f9888532bf1d 2024-11-11T18:13:52,296 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/db63dc40946d40c5bda5043fd3b52e85 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/B/db63dc40946d40c5bda5043fd3b52e85 2024-11-11T18:13:52,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2034576c09494a06819531b2796b9759 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/2034576c09494a06819531b2796b9759 2024-11-11T18:13:52,299 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/66bc9488beb4420192ca78dbc6969575 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/66bc9488beb4420192ca78dbc6969575 2024-11-11T18:13:52,301 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/b11e1482dfe446ee997a0d3c6168d8b8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/b11e1482dfe446ee997a0d3c6168d8b8 2024-11-11T18:13:52,302 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cc21447cf00d4a91a4dc28c4993d3949 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/C/cc21447cf00d4a91a4dc28c4993d3949 2024-11-11T18:13:52,304 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/recovered.edits/704.seqid to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4/recovered.edits/704.seqid 2024-11-11T18:13:52,305 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/2330eb7332d026bb8d820d13c904c2d4 2024-11-11T18:13:52,305 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T18:13:52,310 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=37, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T18:13:52,319 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T18:13:52,349 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T18:13:52,351 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=37, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,351 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T18:13:52,351 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731348832351"}]},"ts":"9223372036854775807"} 2024-11-11T18:13:52,355 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T18:13:52,355 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2330eb7332d026bb8d820d13c904c2d4, NAME => 'TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T18:13:52,355 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T18:13:52,355 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731348832355"}]},"ts":"9223372036854775807"} 2024-11-11T18:13:52,357 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T18:13:52,358 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=37, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-11T18:13:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-11T18:13:52,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T18:13:52,385 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:52,397 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1088901640_22 at /127.0.0.1:45874 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1702191631_22 at /127.0.0.1:45782 [Waiting for operation #303] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1702191631_22 at /127.0.0.1:45958 [Waiting for operation #268] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;90f34c5bee6a:42957-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 319) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8792 (was 8854) 2024-11-11T18:13:52,408 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=503, ProcessCount=11, AvailableMemoryMB=8792 2024-11-11T18:13:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:13:52,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:13:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:52,411 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T18:13:52,412 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:52,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 38 2024-11-11T18:13:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T18:13:52,413 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T18:13:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741976_1152 (size=1061) 2024-11-11T18:13:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T18:13:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T18:13:52,820 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:13:52,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741977_1153 (size=53) 2024-11-11T18:13:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T18:13:53,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:53,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cc678935c00a295255833d490f0a10a8, disabling compactions & flushes 2024-11-11T18:13:53,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. after waiting 0 ms 2024-11-11T18:13:53,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cc678935c00a295255833d490f0a10a8: Waiting for close lock at 1731348833227Disabling compacts and flushes for region at 1731348833227Disabling writes for close at 1731348833227Writing region close event to WAL at 1731348833227Closed at 1731348833227 2024-11-11T18:13:53,228 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T18:13:53,229 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731348833228"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731348833228"}]},"ts":"1731348833228"} 2024-11-11T18:13:53,230 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T18:13:53,231 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T18:13:53,231 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348833231"}]},"ts":"1731348833231"} 2024-11-11T18:13:53,233 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T18:13:53,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, ASSIGN}] 2024-11-11T18:13:53,234 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, ASSIGN 2024-11-11T18:13:53,235 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, ASSIGN; state=OFFLINE, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=false 2024-11-11T18:13:53,385 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:53,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, ASSIGN because future has completed 2024-11-11T18:13:53,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:13:53,543 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,544 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:13:53,544 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,544 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:53,544 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,544 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T18:13:53,545 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,547 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:53,547 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc678935c00a295255833d490f0a10a8 columnFamilyName A 2024-11-11T18:13:53,548 DEBUG [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:53,548 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(327): Store=cc678935c00a295255833d490f0a10a8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:53,548 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,549 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:53,549 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc678935c00a295255833d490f0a10a8 columnFamilyName B 2024-11-11T18:13:53,549 DEBUG [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:53,550 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(327): Store=cc678935c00a295255833d490f0a10a8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:53,550 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,551 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:53,551 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc678935c00a295255833d490f0a10a8 columnFamilyName C 2024-11-11T18:13:53,551 DEBUG [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:53,551 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(327): Store=cc678935c00a295255833d490f0a10a8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:53,551 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,551 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,552 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,552 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,553 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,553 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,554 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:13:53,555 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,556 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:13:53,557 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened cc678935c00a295255833d490f0a10a8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68757678, jitterRate=0.02456924319267273}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:13:53,557 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:53,558 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for cc678935c00a295255833d490f0a10a8: Running coprocessor pre-open hook at 1731348833544Writing region info on filesystem at 1731348833544Initializing all the Stores at 1731348833545 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348833545Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348833545Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348833545Cleaning up temporary data from old regions at 1731348833553 (+8 ms)Running coprocessor post-open hooks at 1731348833557 (+4 ms)Region opened successfully at 1731348833558 (+1 ms) 2024-11-11T18:13:53,559 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., pid=40, masterSystemTime=1731348833540 2024-11-11T18:13:53,561 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,561 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:53,562 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:53,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:13:53,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-11T18:13:53,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 in 176 msec 2024-11-11T18:13:53,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T18:13:53,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, ASSIGN in 333 msec 2024-11-11T18:13:53,569 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T18:13:53,569 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348833569"}]},"ts":"1731348833569"} 2024-11-11T18:13:53,570 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T18:13:53,571 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T18:13:53,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-11T18:13:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T18:13:54,555 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:54,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:54,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:54,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:54,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b76e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:54,560 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478c3bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:54,561 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:54,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:54,565 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dcf76b9 2024-11-11T18:13:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:13:54,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:13:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:13:54,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:13:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T18:13:54,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741978_1154 (size=1097) 2024-11-11T18:13:54,992 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T18:13:54,992 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T18:13:54,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:13:55,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, REOPEN/MOVE}] 2024-11-11T18:13:55,002 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, REOPEN/MOVE 2024-11-11T18:13:55,003 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, REOPEN/MOVE because future has completed 2024-11-11T18:13:55,005 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:13:55,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:13:55,158 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(122): Close cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,158 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:13:55,158 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1722): Closing cc678935c00a295255833d490f0a10a8, disabling compactions & flushes 2024-11-11T18:13:55,158 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,158 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,158 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. after waiting 0 ms 2024-11-11T18:13:55,158 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,162 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T18:13:55,162 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,162 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1676): Region close journal for cc678935c00a295255833d490f0a10a8: Waiting for close lock at 1731348835158Running coprocessor pre-close hooks at 1731348835158Disabling compacts and flushes for region at 1731348835158Disabling writes for close at 1731348835158Writing region close event to WAL at 1731348835158Running coprocessor post-close hooks at 1731348835162 (+4 ms)Closed at 1731348835162 2024-11-11T18:13:55,162 WARN [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegionServer(3299): Not adding moved region record: cc678935c00a295255833d490f0a10a8 to self. 2024-11-11T18:13:55,164 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(157): Closed cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=CLOSED 2024-11-11T18:13:55,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:13:55,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-11T18:13:55,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; CloseRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 in 162 msec 2024-11-11T18:13:55,169 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, REOPEN/MOVE; state=CLOSED, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=true 2024-11-11T18:13:55,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, REOPEN/MOVE because future has completed 2024-11-11T18:13:55,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:13:55,477 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,477 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:13:55,477 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,477 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:13:55,478 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,478 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,482 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,483 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:55,488 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc678935c00a295255833d490f0a10a8 columnFamilyName A 2024-11-11T18:13:55,489 DEBUG [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:55,490 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(327): Store=cc678935c00a295255833d490f0a10a8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:55,490 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,491 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:55,491 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc678935c00a295255833d490f0a10a8 columnFamilyName B 2024-11-11T18:13:55,491 DEBUG [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:55,491 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(327): Store=cc678935c00a295255833d490f0a10a8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:55,492 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,492 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:13:55,492 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cc678935c00a295255833d490f0a10a8 columnFamilyName C 2024-11-11T18:13:55,492 DEBUG [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:55,493 INFO [StoreOpener-cc678935c00a295255833d490f0a10a8-1 {}] regionserver.HStore(327): Store=cc678935c00a295255833d490f0a10a8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:13:55,493 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,493 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,493 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,494 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,495 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,495 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,496 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:13:55,497 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,497 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened cc678935c00a295255833d490f0a10a8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66978793, jitterRate=-0.0019382089376449585}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:13:55,498 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,499 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for cc678935c00a295255833d490f0a10a8: Running coprocessor pre-open hook at 1731348835478Writing region info on filesystem at 1731348835478Initializing all the Stores at 1731348835478Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348835478Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348835479 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348835479Cleaning up temporary data from old regions at 1731348835495 (+16 ms)Running coprocessor post-open hooks at 1731348835498 (+3 ms)Region opened successfully at 1731348835498 2024-11-11T18:13:55,499 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., pid=45, masterSystemTime=1731348835474 2024-11-11T18:13:55,501 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,501 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=OPEN, openSeqNum=5, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:13:55,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=43 2024-11-11T18:13:55,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=43, state=SUCCESS, hasLock=false; OpenRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 in 182 msec 2024-11-11T18:13:55,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T18:13:55,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, REOPEN/MOVE in 505 msec 2024-11-11T18:13:55,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-11T18:13:55,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-11T18:13:55,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-11T18:13:55,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-11-11T18:13:55,515 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:55,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6f50f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3ef9d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,521 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,521 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,522 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cc242b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,523 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8bd628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,525 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a7afe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,527 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d0d155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,529 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6779215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,530 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54559565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc72848, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,534 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51121548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,536 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5016e5a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,537 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5609cb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,539 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6100fcb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,540 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1582771d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,542 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad0a601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,544 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787a4f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:13:55,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:13:55,546 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:13:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:13:55,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:13:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218019a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:13:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:13:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:13:55,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:13:55,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9c90f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:13:55,549 DEBUG [Thread-747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,549 DEBUG [Thread-758 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,550 DEBUG [Thread-764 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,550 DEBUG [Thread-755 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,551 DEBUG [Thread-762 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,551 DEBUG [Thread-749 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,551 DEBUG [Thread-753 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,551 DEBUG [Thread-760 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T18:13:55,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,555 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:55,556 DEBUG [Thread-751 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:13:55,556 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:13:55,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:55,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:13:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:13:55,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:13:55,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:13:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:13:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:13:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:13:55,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:55,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111299a7dbb33f94948997b82dbcbdc10d6_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348835574/Put/seqid=0 2024-11-11T18:13:55,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741979_1155 (size=14594) 2024-11-11T18:13:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T18:13:55,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348835623, max timestamp=1731348835668Num uniques 859; 2024-11-11T18:13:55,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:55,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348835623, max timestamp=1731348835668Num uniques 859; 2024-11-11T18:13:55,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:55,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348835623, max timestamp=1731348835668Num uniques 859; 2024-11-11T18:13:55,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:55,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T18:13:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:55,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348895705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348895705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348895712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348895715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348895715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348895825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348895825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348895826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348895835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348895835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:55,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:55,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T18:13:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T18:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:55,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:55,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T18:13:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:56,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:56,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111299a7dbb33f94948997b82dbcbdc10d6_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111299a7dbb33f94948997b82dbcbdc10d6_cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:56,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5437754b4f404cba8cf40dc1abc970ec, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348896046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348896046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348896047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348896049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348896049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5437754b4f404cba8cf40dc1abc970ec is 175, key is test_row_0/A:col10/1731348835574/Put/seqid=0 2024-11-11T18:13:56,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741980_1156 (size=39549) 2024-11-11T18:13:56,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5437754b4f404cba8cf40dc1abc970ec 2024-11-11T18:13:56,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/6f7430d12ef94ef8b07688047f164852 is 50, key is test_row_0/B:col10/1731348835574/Put/seqid=0 2024-11-11T18:13:56,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741981_1157 (size=12001) 2024-11-11T18:13:56,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/6f7430d12ef94ef8b07688047f164852 2024-11-11T18:13:56,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/858b78358fd34e97a7af132b80a60c1e is 50, key is test_row_0/C:col10/1731348835574/Put/seqid=0 2024-11-11T18:13:56,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741982_1158 (size=12001) 2024-11-11T18:13:56,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/858b78358fd34e97a7af132b80a60c1e 2024-11-11T18:13:56,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5437754b4f404cba8cf40dc1abc970ec as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec 2024-11-11T18:13:56,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T18:13:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:56,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec, entries=200, sequenceid=15, filesize=38.6 K 2024-11-11T18:13:56,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:56,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T18:13:56,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/6f7430d12ef94ef8b07688047f164852 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/6f7430d12ef94ef8b07688047f164852 2024-11-11T18:13:56,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/6f7430d12ef94ef8b07688047f164852, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T18:13:56,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/858b78358fd34e97a7af132b80a60c1e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/858b78358fd34e97a7af132b80a60c1e 2024-11-11T18:13:56,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/858b78358fd34e97a7af132b80a60c1e, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T18:13:56,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cc678935c00a295255833d490f0a10a8 in 616ms, sequenceid=15, compaction requested=false 2024-11-11T18:13:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T18:13:56,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:56,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T18:13:56,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:56,327 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T18:13:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:13:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:13:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:13:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:13:56,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e073fc9b58dc4a818d12ba7d775689f8_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348835704/Put/seqid=0 2024-11-11T18:13:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741983_1159 (size=12154) 2024-11-11T18:13:56,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:56,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348896365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348896366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348896370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348896370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348896372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348896475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348896475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348896484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348896484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348896495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T18:13:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348896684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348896685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348896694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348896704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348896704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:56,768 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e073fc9b58dc4a818d12ba7d775689f8_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e073fc9b58dc4a818d12ba7d775689f8_cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:56,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0a4e678e0d6946bcacc76160748626f5, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:56,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0a4e678e0d6946bcacc76160748626f5 is 175, key is test_row_0/A:col10/1731348835704/Put/seqid=0 2024-11-11T18:13:56,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741984_1160 (size=30955) 2024-11-11T18:13:56,778 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0a4e678e0d6946bcacc76160748626f5 2024-11-11T18:13:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/ee9ebce9116f46169ca0da8865527848 is 50, key is test_row_0/B:col10/1731348835704/Put/seqid=0 2024-11-11T18:13:56,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741985_1161 (size=12001) 2024-11-11T18:13:56,799 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/ee9ebce9116f46169ca0da8865527848 2024-11-11T18:13:56,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/ca2faecf23804e17bcef38ae28470065 is 50, key is test_row_0/C:col10/1731348835704/Put/seqid=0 2024-11-11T18:13:56,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741986_1162 (size=12001) 2024-11-11T18:13:56,820 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/ca2faecf23804e17bcef38ae28470065 2024-11-11T18:13:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0a4e678e0d6946bcacc76160748626f5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5 2024-11-11T18:13:56,839 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5, entries=150, sequenceid=51, filesize=30.2 K 2024-11-11T18:13:56,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/ee9ebce9116f46169ca0da8865527848 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ee9ebce9116f46169ca0da8865527848 2024-11-11T18:13:56,846 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ee9ebce9116f46169ca0da8865527848, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T18:13:56,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/ca2faecf23804e17bcef38ae28470065 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ca2faecf23804e17bcef38ae28470065 2024-11-11T18:13:56,853 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ca2faecf23804e17bcef38ae28470065, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T18:13:56,855 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cc678935c00a295255833d490f0a10a8 in 528ms, sequenceid=51, compaction requested=false 2024-11-11T18:13:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T18:13:56,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T18:13:56,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T18:13:56,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3000 sec 2024-11-11T18:13:56,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3090 sec 2024-11-11T18:13:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:56,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:13:57,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:13:57,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:57,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:13:57,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:57,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:13:57,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:57,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111124f3dff788de4d4980b8a252f8806e44_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741987_1163 (size=12154) 2024-11-11T18:13:57,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348897027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348897029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348897030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348897030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348897038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348897146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348897146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348897152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348897152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348897154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,248 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:13:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348897354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348897355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348897374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348897374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348897374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,436 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:57,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111124f3dff788de4d4980b8a252f8806e44_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124f3dff788de4d4980b8a252f8806e44_cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:57,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/95f8cbba22044e9c96b6d7c9f3d1f3e0, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:57,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/95f8cbba22044e9c96b6d7c9f3d1f3e0 is 175, key is test_row_0/A:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741988_1164 (size=30955) 2024-11-11T18:13:57,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/95f8cbba22044e9c96b6d7c9f3d1f3e0 2024-11-11T18:13:57,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/4fb792b1381e4c3d9c8bfbb7749188e9 is 50, key is test_row_0/B:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741989_1165 (size=12001) 2024-11-11T18:13:57,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/4fb792b1381e4c3d9c8bfbb7749188e9 2024-11-11T18:13:57,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/49f454527a80467ea267c238948b47c8 is 50, key is test_row_0/C:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741990_1166 (size=12001) 2024-11-11T18:13:57,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/49f454527a80467ea267c238948b47c8 2024-11-11T18:13:57,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/95f8cbba22044e9c96b6d7c9f3d1f3e0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0 2024-11-11T18:13:57,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0, entries=150, sequenceid=64, filesize=30.2 K 2024-11-11T18:13:57,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/4fb792b1381e4c3d9c8bfbb7749188e9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4fb792b1381e4c3d9c8bfbb7749188e9 2024-11-11T18:13:57,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4fb792b1381e4c3d9c8bfbb7749188e9, entries=150, sequenceid=64, filesize=11.7 K 2024-11-11T18:13:57,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/49f454527a80467ea267c238948b47c8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/49f454527a80467ea267c238948b47c8 2024-11-11T18:13:57,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/49f454527a80467ea267c238948b47c8, entries=150, sequenceid=64, filesize=11.7 K 2024-11-11T18:13:57,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cc678935c00a295255833d490f0a10a8 in 538ms, sequenceid=64, compaction requested=true 2024-11-11T18:13:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:57,537 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:13:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:57,538 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:13:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:13:57,539 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:57,539 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:13:57,539 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:57,539 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=99.1 K 2024-11-11T18:13:57,539 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:57,539 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0] 2024-11-11T18:13:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:57,540 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5437754b4f404cba8cf40dc1abc970ec, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731348835566 2024-11-11T18:13:57,541 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:57,541 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:13:57,541 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:57,541 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/6f7430d12ef94ef8b07688047f164852, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ee9ebce9116f46169ca0da8865527848, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4fb792b1381e4c3d9c8bfbb7749188e9] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=35.2 K 2024-11-11T18:13:57,541 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4e678e0d6946bcacc76160748626f5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731348835704 2024-11-11T18:13:57,543 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7430d12ef94ef8b07688047f164852, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731348835566 2024-11-11T18:13:57,543 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f8cbba22044e9c96b6d7c9f3d1f3e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731348836363 2024-11-11T18:13:57,543 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ee9ebce9116f46169ca0da8865527848, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731348835704 2024-11-11T18:13:57,543 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb792b1381e4c3d9c8bfbb7749188e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731348836363 2024-11-11T18:13:57,567 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:57,568 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f386b144f6c944d0b30a97af2351425b is 50, key is test_row_0/B:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,574 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:57,603 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d587b355bbd447b7947afc3044525d5d_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:57,608 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d587b355bbd447b7947afc3044525d5d_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:57,608 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d587b355bbd447b7947afc3044525d5d_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741991_1167 (size=12104) 2024-11-11T18:13:57,622 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f386b144f6c944d0b30a97af2351425b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f386b144f6c944d0b30a97af2351425b 2024-11-11T18:13:57,635 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into f386b144f6c944d0b30a97af2351425b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:57,636 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:57,636 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=13, startTime=1731348837538; duration=0sec 2024-11-11T18:13:57,637 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:13:57,637 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:13:57,637 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:13:57,641 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:13:57,641 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:13:57,641 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:57,642 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/858b78358fd34e97a7af132b80a60c1e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ca2faecf23804e17bcef38ae28470065, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/49f454527a80467ea267c238948b47c8] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=35.2 K 2024-11-11T18:13:57,642 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 858b78358fd34e97a7af132b80a60c1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731348835566 2024-11-11T18:13:57,643 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2faecf23804e17bcef38ae28470065, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731348835704 2024-11-11T18:13:57,644 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f454527a80467ea267c238948b47c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731348836363 2024-11-11T18:13:57,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741992_1168 (size=4469) 2024-11-11T18:13:57,650 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#149 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:57,651 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/4521633d6e9e4fe9a318cb4a3862234b is 175, key is test_row_0/A:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,659 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:13:57,660 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/f510061cdd1d4bb7beb753cad85d5e9b is 50, key is test_row_0/C:col10/1731348836999/Put/seqid=0 2024-11-11T18:13:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:57,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:13:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:13:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:13:57,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:57,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:13:57,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:57,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741993_1169 (size=31058) 2024-11-11T18:13:57,683 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/4521633d6e9e4fe9a318cb4a3862234b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/4521633d6e9e4fe9a318cb4a3862234b 2024-11-11T18:13:57,690 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into 4521633d6e9e4fe9a318cb4a3862234b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:57,690 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:57,690 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=13, startTime=1731348837537; duration=0sec 2024-11-11T18:13:57,690 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:57,690 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:13:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T18:13:57,695 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:13:57,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:13:57,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348897727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348897727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T18:13:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348897690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741994_1170 (size=12104) 2024-11-11T18:13:57,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348897735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:13:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348897735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:13:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,740 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:13:57,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:13:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c91e666de8fc49feaa097da81fc9a987_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348837028/Put/seqid=0 2024-11-11T18:13:57,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741995_1171 (size=14594) 2024-11-11T18:13:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:13:57,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348897845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348897845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348897846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348897846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348897846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:57,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:57,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T18:13:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:57,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T18:13:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:58,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:13:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348898065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348898066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348898066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348898066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348898066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,136 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/f510061cdd1d4bb7beb753cad85d5e9b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/f510061cdd1d4bb7beb753cad85d5e9b 2024-11-11T18:13:58,144 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into f510061cdd1d4bb7beb753cad85d5e9b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:13:58,144 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:58,144 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=13, startTime=1731348837538; duration=0sec 2024-11-11T18:13:58,144 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:13:58,144 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:13:58,198 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:58,203 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c91e666de8fc49feaa097da81fc9a987_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c91e666de8fc49feaa097da81fc9a987_cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:58,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T18:13:58,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:58,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:58,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:58,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,204 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0c3c839a1909406e9bf67c1d1f8aa132, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:13:58,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0c3c839a1909406e9bf67c1d1f8aa132 is 175, key is test_row_0/A:col10/1731348837028/Put/seqid=0 2024-11-11T18:13:58,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741996_1172 (size=39549) 2024-11-11T18:13:58,213 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0c3c839a1909406e9bf67c1d1f8aa132 2024-11-11T18:13:58,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/c0e1ceeb1ac1460db1a988bdd24dca1f is 50, key is test_row_0/B:col10/1731348837028/Put/seqid=0 2024-11-11T18:13:58,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741997_1173 (size=12001) 2024-11-11T18:13:58,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/c0e1ceeb1ac1460db1a988bdd24dca1f 2024-11-11T18:13:58,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/14ccfab7744e49a4890c54ffb979bf75 is 50, key is test_row_0/C:col10/1731348837028/Put/seqid=0 2024-11-11T18:13:58,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741998_1174 (size=12001) 2024-11-11T18:13:58,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/14ccfab7744e49a4890c54ffb979bf75 2024-11-11T18:13:58,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/0c3c839a1909406e9bf67c1d1f8aa132 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132 2024-11-11T18:13:58,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132, entries=200, sequenceid=85, filesize=38.6 K 2024-11-11T18:13:58,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/c0e1ceeb1ac1460db1a988bdd24dca1f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c0e1ceeb1ac1460db1a988bdd24dca1f 2024-11-11T18:13:58,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c0e1ceeb1ac1460db1a988bdd24dca1f, entries=150, sequenceid=85, filesize=11.7 K 2024-11-11T18:13:58,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/14ccfab7744e49a4890c54ffb979bf75 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/14ccfab7744e49a4890c54ffb979bf75 2024-11-11T18:13:58,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/14ccfab7744e49a4890c54ffb979bf75, entries=150, sequenceid=85, filesize=11.7 K 2024-11-11T18:13:58,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cc678935c00a295255833d490f0a10a8 in 679ms, sequenceid=85, compaction requested=false 2024-11-11T18:13:58,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:58,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T18:13:58,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:58,357 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:13:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:13:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:13:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:13:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:13:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:13:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:58,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:13:58,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111166d727d2a41b48f19db992172e47d899_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348837725/Put/seqid=0 2024-11-11T18:13:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348898433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348898434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348838408, max timestamp=1731348838435Num uniques 859; 2024-11-11T18:13:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348838408, max timestamp=1731348838435Num uniques 859; 2024-11-11T18:13:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:58,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:58,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741999_1175 (size=12154) 2024-11-11T18:13:58,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348838408, max timestamp=1731348838435Num uniques 859; 2024-11-11T18:13:58,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:13:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348898555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348898555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348898568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348898568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348898576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348898699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348898699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348898699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348898794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348898794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:13:58,853 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111166d727d2a41b48f19db992172e47d899_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166d727d2a41b48f19db992172e47d899_cc678935c00a295255833d490f0a10a8 2024-11-11T18:13:58,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/88c2abcb8db04ff5bad30282ae760209, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:13:58,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/88c2abcb8db04ff5bad30282ae760209 is 175, key is test_row_0/A:col10/1731348837725/Put/seqid=0 2024-11-11T18:13:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:13:58,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742000_1176 (size=30955) 2024-11-11T18:13:58,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348898914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348898914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:58,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348898914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348899114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348899114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348899234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348899235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348899235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,309 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/88c2abcb8db04ff5bad30282ae760209 2024-11-11T18:13:59,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/d24d2585af4b4f55ae7bbeb8454897c1 is 50, key is test_row_0/B:col10/1731348837725/Put/seqid=0 2024-11-11T18:13:59,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742001_1177 (size=12001) 2024-11-11T18:13:59,374 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/d24d2585af4b4f55ae7bbeb8454897c1 2024-11-11T18:13:59,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/10a3dc191ead46239079ef798c77cfa3 is 50, key is test_row_0/C:col10/1731348837725/Put/seqid=0 2024-11-11T18:13:59,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742002_1178 (size=12001) 2024-11-11T18:13:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348899624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348899624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348899745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348899745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348899745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:13:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:13:59,845 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/10a3dc191ead46239079ef798c77cfa3 2024-11-11T18:13:59,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/88c2abcb8db04ff5bad30282ae760209 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209 2024-11-11T18:13:59,857 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209, entries=150, sequenceid=99, filesize=30.2 K 2024-11-11T18:13:59,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/d24d2585af4b4f55ae7bbeb8454897c1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/d24d2585af4b4f55ae7bbeb8454897c1 2024-11-11T18:13:59,881 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/d24d2585af4b4f55ae7bbeb8454897c1, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T18:13:59,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/10a3dc191ead46239079ef798c77cfa3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/10a3dc191ead46239079ef798c77cfa3 2024-11-11T18:13:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:13:59,893 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/10a3dc191ead46239079ef798c77cfa3, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T18:13:59,903 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for cc678935c00a295255833d490f0a10a8 in 1546ms, sequenceid=99, compaction requested=true 2024-11-11T18:13:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:13:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:13:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T18:13:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T18:13:59,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T18:13:59,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1660 sec 2024-11-11T18:13:59,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2140 sec 2024-11-11T18:14:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:00,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T18:14:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:00,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111719234609e4d403396f1f0b7a3118baa_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742003_1179 (size=14744) 2024-11-11T18:14:00,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348900685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348900686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348900754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348900755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,755 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:00,756 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:00,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348900765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,769 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:00,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:00,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348900794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348901005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348901005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:01,083 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111719234609e4d403396f1f0b7a3118baa_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111719234609e4d403396f1f0b7a3118baa_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:01,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/251cb6dc7f7841b985aca9cc169b2925, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:01,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/251cb6dc7f7841b985aca9cc169b2925 is 175, key is test_row_0/A:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:01,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742004_1180 (size=39699) 2024-11-11T18:14:01,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/251cb6dc7f7841b985aca9cc169b2925 2024-11-11T18:14:01,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f691c60faa5c433a9fd77a1ca772ea6f is 50, key is test_row_0/B:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:01,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742005_1181 (size=12101) 2024-11-11T18:14:01,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348901314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:01,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348901314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:01,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:01,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f691c60faa5c433a9fd77a1ca772ea6f 2024-11-11T18:14:01,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8dc9aeea8de94c3595b3636fa60af3c1 is 50, key is test_row_0/C:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:01,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742006_1182 (size=12101) 2024-11-11T18:14:01,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8dc9aeea8de94c3595b3636fa60af3c1 2024-11-11T18:14:01,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/251cb6dc7f7841b985aca9cc169b2925 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925 2024-11-11T18:14:01,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925, entries=200, sequenceid=136, filesize=38.8 K 2024-11-11T18:14:01,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f691c60faa5c433a9fd77a1ca772ea6f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f691c60faa5c433a9fd77a1ca772ea6f 2024-11-11T18:14:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f691c60faa5c433a9fd77a1ca772ea6f, entries=150, sequenceid=136, filesize=11.8 K 2024-11-11T18:14:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8dc9aeea8de94c3595b3636fa60af3c1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8dc9aeea8de94c3595b3636fa60af3c1 2024-11-11T18:14:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8dc9aeea8de94c3595b3636fa60af3c1, entries=150, sequenceid=136, filesize=11.8 K 2024-11-11T18:14:01,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cc678935c00a295255833d490f0a10a8 in 985ms, sequenceid=136, compaction requested=true 2024-11-11T18:14:01,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,643 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,645 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141261 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:01,645 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:01,645 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,646 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/4521633d6e9e4fe9a318cb4a3862234b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=138.0 K 2024-11-11T18:14:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,646 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:01,646 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/4521633d6e9e4fe9a318cb4a3862234b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925] 2024-11-11T18:14:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,647 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4521633d6e9e4fe9a318cb4a3862234b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731348836363 2024-11-11T18:14:01,647 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3c839a1909406e9bf67c1d1f8aa132, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731348837023 2024-11-11T18:14:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,648 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88c2abcb8db04ff5bad30282ae760209, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731348837694 2024-11-11T18:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:01,648 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 251cb6dc7f7841b985aca9cc169b2925, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731348838498 2024-11-11T18:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:01,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,650 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:01,651 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,651 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:01,651 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f386b144f6c944d0b30a97af2351425b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c0e1ceeb1ac1460db1a988bdd24dca1f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/d24d2585af4b4f55ae7bbeb8454897c1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f691c60faa5c433a9fd77a1ca772ea6f] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=47.1 K 2024-11-11T18:14:01,651 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f386b144f6c944d0b30a97af2351425b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731348836363 2024-11-11T18:14:01,652 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e1ceeb1ac1460db1a988bdd24dca1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731348837028 2024-11-11T18:14:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,652 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d24d2585af4b4f55ae7bbeb8454897c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731348837694 2024-11-11T18:14:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,653 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f691c60faa5c433a9fd77a1ca772ea6f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731348838498 2024-11-11T18:14:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,670 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,683 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,684 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/422483e769b043f09dcfaa9d908e0508 is 50, key is test_row_0/B:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,695 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a6d2ade312804d9fac62aefd85dff775_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,699 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a6d2ade312804d9fac62aefd85dff775_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:01,699 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a6d2ade312804d9fac62aefd85dff775_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742007_1183 (size=12341) 2024-11-11T18:14:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,729 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/422483e769b043f09dcfaa9d908e0508 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/422483e769b043f09dcfaa9d908e0508 2024-11-11T18:14:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,736 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into 422483e769b043f09dcfaa9d908e0508(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:01,736 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:01,736 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=12, startTime=1731348841648; duration=0sec 2024-11-11T18:14:01,737 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:01,737 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,737 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,739 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:01,739 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:14:01,739 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:01,739 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/f510061cdd1d4bb7beb753cad85d5e9b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/14ccfab7744e49a4890c54ffb979bf75, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/10a3dc191ead46239079ef798c77cfa3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8dc9aeea8de94c3595b3636fa60af3c1] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=47.1 K 2024-11-11T18:14:01,739 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f510061cdd1d4bb7beb753cad85d5e9b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731348836363 2024-11-11T18:14:01,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742008_1184 (size=4469) 2024-11-11T18:14:01,740 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ccfab7744e49a4890c54ffb979bf75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731348837028 2024-11-11T18:14:01,740 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a3dc191ead46239079ef798c77cfa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731348837694 2024-11-11T18:14:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,741 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc9aeea8de94c3595b3636fa60af3c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731348838498 2024-11-11T18:14:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,742 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#160 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:01,743 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/91ca3b2bb8e147e98cfbe9346f02da80 is 175, key is test_row_0/A:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742009_1185 (size=31295) 2024-11-11T18:14:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,778 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/91ca3b2bb8e147e98cfbe9346f02da80 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91ca3b2bb8e147e98cfbe9346f02da80 2024-11-11T18:14:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,780 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#162 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,781 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/32bbe863f1ee4a93b1b38019650764a2 is 50, key is test_row_0/C:col10/1731348840657/Put/seqid=0 2024-11-11T18:14:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,784 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into 91ca3b2bb8e147e98cfbe9346f02da80(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:01,784 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:01,784 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=12, startTime=1731348841643; duration=0sec 2024-11-11T18:14:01,785 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:01,785 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742010_1186 (size=12341) 2024-11-11T18:14:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:01,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:01,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:01,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:01,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:01,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:01,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,835 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/32bbe863f1ee4a93b1b38019650764a2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/32bbe863f1ee4a93b1b38019650764a2 2024-11-11T18:14:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,841 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into 32bbe863f1ee4a93b1b38019650764a2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:01,841 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:01,841 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=12, startTime=1731348841648; duration=0sec 2024-11-11T18:14:01,841 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:01,841 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111128e8b2ceec3541179231bcaa21f5d962_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348840685/Put/seqid=0 2024-11-11T18:14:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742011_1187 (size=12304) 2024-11-11T18:14:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T18:14:01,905 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:01,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T18:14:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:01,910 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:01,911 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:01,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:01,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348841866, max timestamp=1731348841905Num uniques 859; 2024-11-11T18:14:01,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:01,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348841866, max timestamp=1731348841905Num uniques 859; 2024-11-11T18:14:01,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:01,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348841866, max timestamp=1731348841905Num uniques 859; 2024-11-11T18:14:01,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:02,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348902016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348902017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348902134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348902134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:02,259 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:02,264 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111128e8b2ceec3541179231bcaa21f5d962_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111128e8b2ceec3541179231bcaa21f5d962_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:02,265 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:02,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff is 175, key is test_row_0/A:col10/1731348840685/Put/seqid=0 2024-11-11T18:14:02,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742012_1188 (size=31101) 2024-11-11T18:14:02,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348902348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348902348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:02,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348902655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348902665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff 2024-11-11T18:14:02,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/53952ba289194bfe9daf40fd25a1b2ed is 50, key is test_row_0/B:col10/1731348840685/Put/seqid=0 2024-11-11T18:14:02,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742013_1189 (size=9757) 2024-11-11T18:14:02,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348902774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348902775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,778 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,778 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348902795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:02,801 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:02,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:02,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:02,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:02,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:02,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:02,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:03,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/53952ba289194bfe9daf40fd25a1b2ed 2024-11-11T18:14:03,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/b075e4f871e84dd6be118c3fdec5c8c7 is 50, key is test_row_0/C:col10/1731348840685/Put/seqid=0 2024-11-11T18:14:03,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742014_1190 (size=9757) 2024-11-11T18:14:03,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:03,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:03,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348903164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348903176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:03,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:03,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:03,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:03,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:03,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/b075e4f871e84dd6be118c3fdec5c8c7 2024-11-11T18:14:03,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff 2024-11-11T18:14:03,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff, entries=150, sequenceid=149, filesize=30.4 K 2024-11-11T18:14:03,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/53952ba289194bfe9daf40fd25a1b2ed as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/53952ba289194bfe9daf40fd25a1b2ed 2024-11-11T18:14:03,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/53952ba289194bfe9daf40fd25a1b2ed, entries=100, sequenceid=149, filesize=9.5 K 2024-11-11T18:14:03,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/b075e4f871e84dd6be118c3fdec5c8c7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b075e4f871e84dd6be118c3fdec5c8c7 2024-11-11T18:14:03,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b075e4f871e84dd6be118c3fdec5c8c7, entries=100, sequenceid=149, filesize=9.5 K 2024-11-11T18:14:03,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cc678935c00a295255833d490f0a10a8 in 1755ms, sequenceid=149, compaction requested=false 2024-11-11T18:14:03,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:03,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T18:14:03,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:03,608 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T18:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:03,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114445a8ac3b3a465895abdc41f39bf8ef_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348842013/Put/seqid=0 2024-11-11T18:14:03,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742015_1191 (size=12304) 2024-11-11T18:14:04,035 INFO [master/90f34c5bee6a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T18:14:04,035 INFO [master/90f34c5bee6a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T18:14:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:04,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:04,070 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114445a8ac3b3a465895abdc41f39bf8ef_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114445a8ac3b3a465895abdc41f39bf8ef_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/218a6914cf284f3a8dba9d6dec510bc9, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:04,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/218a6914cf284f3a8dba9d6dec510bc9 is 175, key is test_row_0/A:col10/1731348842013/Put/seqid=0 2024-11-11T18:14:04,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742016_1192 (size=31105) 2024-11-11T18:14:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:04,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348904219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348904222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348904336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348904336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,483 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/218a6914cf284f3a8dba9d6dec510bc9 2024-11-11T18:14:04,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/dfd2be811a924a3eba272ab5eb41d91c is 50, key is test_row_0/B:col10/1731348842013/Put/seqid=0 2024-11-11T18:14:04,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742017_1193 (size=12151) 2024-11-11T18:14:04,508 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/dfd2be811a924a3eba272ab5eb41d91c 2024-11-11T18:14:04,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8039e2b059964a67bbe045cd6290fe1a is 50, key is test_row_0/C:col10/1731348842013/Put/seqid=0 2024-11-11T18:14:04,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742018_1194 (size=12151) 2024-11-11T18:14:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348904544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348904554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348904855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348904864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:04,923 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8039e2b059964a67bbe045cd6290fe1a 2024-11-11T18:14:04,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/218a6914cf284f3a8dba9d6dec510bc9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9 2024-11-11T18:14:04,934 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9, entries=150, sequenceid=185, filesize=30.4 K 2024-11-11T18:14:04,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/dfd2be811a924a3eba272ab5eb41d91c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dfd2be811a924a3eba272ab5eb41d91c 2024-11-11T18:14:04,940 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dfd2be811a924a3eba272ab5eb41d91c, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T18:14:04,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8039e2b059964a67bbe045cd6290fe1a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8039e2b059964a67bbe045cd6290fe1a 2024-11-11T18:14:04,955 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8039e2b059964a67bbe045cd6290fe1a, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T18:14:04,956 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for cc678935c00a295255833d490f0a10a8 in 1349ms, sequenceid=185, compaction requested=true 2024-11-11T18:14:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:04,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T18:14:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T18:14:04,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T18:14:04,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0470 sec 2024-11-11T18:14:04,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.0530 sec 2024-11-11T18:14:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:05,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:14:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:05,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:05,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f914931a7e4d45de922404d87b8aa60c_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:05,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742019_1195 (size=14794) 2024-11-11T18:14:05,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:05,418 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f914931a7e4d45de922404d87b8aa60c_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f914931a7e4d45de922404d87b8aa60c_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:05,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/91b62bb75f574b32b511ddb5c6030ffe, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:05,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/91b62bb75f574b32b511ddb5c6030ffe is 175, key is test_row_0/A:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:05,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742020_1196 (size=39749) 2024-11-11T18:14:05,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/91b62bb75f574b32b511ddb5c6030ffe 2024-11-11T18:14:05,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/2b143124fbcf423091725b748dd76824 is 50, key is test_row_0/B:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:05,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348905445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:05,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348905453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:05,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742021_1197 (size=12151) 2024-11-11T18:14:05,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348905564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:05,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348905565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:05,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348905774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:05,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348905775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:05,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:05,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/2b143124fbcf423091725b748dd76824 2024-11-11T18:14:05,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7bc2acc30ce64e01b5411812d4c1b33f is 50, key is test_row_0/C:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:05,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742022_1198 (size=12151) 2024-11-11T18:14:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T18:14:06,075 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:06,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T18:14:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T18:14:06,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:06,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:06,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348906084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348906084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T18:14:06,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T18:14:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:06,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:06,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:06,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7bc2acc30ce64e01b5411812d4c1b33f 2024-11-11T18:14:06,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/91b62bb75f574b32b511ddb5c6030ffe as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe 2024-11-11T18:14:06,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe, entries=200, sequenceid=197, filesize=38.8 K 2024-11-11T18:14:06,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/2b143124fbcf423091725b748dd76824 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/2b143124fbcf423091725b748dd76824 2024-11-11T18:14:06,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/2b143124fbcf423091725b748dd76824, entries=150, sequenceid=197, filesize=11.9 K 2024-11-11T18:14:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7bc2acc30ce64e01b5411812d4c1b33f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7bc2acc30ce64e01b5411812d4c1b33f 2024-11-11T18:14:06,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7bc2acc30ce64e01b5411812d4c1b33f, entries=150, sequenceid=197, filesize=11.9 K 2024-11-11T18:14:06,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cc678935c00a295255833d490f0a10a8 in 949ms, sequenceid=197, compaction requested=true 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:06,318 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:06,318 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:06,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:06,320 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133250 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:06,320 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:06,320 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,320 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91ca3b2bb8e147e98cfbe9346f02da80, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=130.1 K 2024-11-11T18:14:06,321 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,321 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91ca3b2bb8e147e98cfbe9346f02da80, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe] 2024-11-11T18:14:06,324 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:06,324 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:06,324 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,324 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/422483e769b043f09dcfaa9d908e0508, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/53952ba289194bfe9daf40fd25a1b2ed, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dfd2be811a924a3eba272ab5eb41d91c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/2b143124fbcf423091725b748dd76824] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=45.3 K 2024-11-11T18:14:06,325 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ca3b2bb8e147e98cfbe9346f02da80, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731348838498 2024-11-11T18:14:06,325 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 422483e769b043f09dcfaa9d908e0508, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731348838498 2024-11-11T18:14:06,325 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf3b14ee00c4d57ad2ee0f6e8b7a0ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731348840674 2024-11-11T18:14:06,326 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 53952ba289194bfe9daf40fd25a1b2ed, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731348840685 2024-11-11T18:14:06,326 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218a6914cf284f3a8dba9d6dec510bc9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731348841993 2024-11-11T18:14:06,326 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b62bb75f574b32b511ddb5c6030ffe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731348844209 2024-11-11T18:14:06,326 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd2be811a924a3eba272ab5eb41d91c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731348841993 2024-11-11T18:14:06,326 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b143124fbcf423091725b748dd76824, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731348844211 2024-11-11T18:14:06,338 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:06,339 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/36b3c5ee684b43759cd628f5a055c5c3 is 50, key is test_row_0/B:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:06,339 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:06,347 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110c597f32255340d0bc602a76e4011be0_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:06,349 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110c597f32255340d0bc602a76e4011be0_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:06,349 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110c597f32255340d0bc602a76e4011be0_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:06,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742023_1199 (size=12527) 2024-11-11T18:14:06,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742024_1200 (size=4469) 2024-11-11T18:14:06,360 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#173 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:06,360 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/8b8d577c0b5849df98373dd1ea162dcd is 175, key is test_row_0/A:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:06,365 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/36b3c5ee684b43759cd628f5a055c5c3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/36b3c5ee684b43759cd628f5a055c5c3 2024-11-11T18:14:06,370 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into 36b3c5ee684b43759cd628f5a055c5c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:06,370 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:06,370 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=12, startTime=1731348846318; duration=0sec 2024-11-11T18:14:06,371 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:06,371 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:06,371 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:06,372 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:06,372 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:14:06,372 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,372 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/32bbe863f1ee4a93b1b38019650764a2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b075e4f871e84dd6be118c3fdec5c8c7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8039e2b059964a67bbe045cd6290fe1a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7bc2acc30ce64e01b5411812d4c1b33f] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=45.3 K 2024-11-11T18:14:06,372 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 32bbe863f1ee4a93b1b38019650764a2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731348838498 2024-11-11T18:14:06,374 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b075e4f871e84dd6be118c3fdec5c8c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731348840685 2024-11-11T18:14:06,375 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8039e2b059964a67bbe045cd6290fe1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731348841993 2024-11-11T18:14:06,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742025_1201 (size=31481) 2024-11-11T18:14:06,375 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bc2acc30ce64e01b5411812d4c1b33f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731348844211 2024-11-11T18:14:06,383 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/8b8d577c0b5849df98373dd1ea162dcd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8b8d577c0b5849df98373dd1ea162dcd 2024-11-11T18:14:06,387 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:06,387 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/3b9190fe6b06485e81b59d2bad69bc77 is 50, key is test_row_0/C:col10/1731348844219/Put/seqid=0 2024-11-11T18:14:06,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T18:14:06,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,388 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:14:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:06,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T18:14:06,395 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into 8b8d577c0b5849df98373dd1ea162dcd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:06,395 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:06,396 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=12, startTime=1731348846318; duration=0sec 2024-11-11T18:14:06,396 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:06,396 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:06,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742026_1202 (size=12527) 2024-11-11T18:14:06,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111181440c46844b494d9fd2f2c3b67d518f_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348845442/Put/seqid=0 2024-11-11T18:14:06,422 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/3b9190fe6b06485e81b59d2bad69bc77 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/3b9190fe6b06485e81b59d2bad69bc77 2024-11-11T18:14:06,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742027_1203 (size=12304) 2024-11-11T18:14:06,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:06,440 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111181440c46844b494d9fd2f2c3b67d518f_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111181440c46844b494d9fd2f2c3b67d518f_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:06,441 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into 3b9190fe6b06485e81b59d2bad69bc77(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:06,441 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:06,441 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=12, startTime=1731348846318; duration=0sec 2024-11-11T18:14:06,441 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:06,441 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:06,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5c7af1c27173434c8493cac5bd8458a5, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5c7af1c27173434c8493cac5bd8458a5 is 175, key is test_row_0/A:col10/1731348845442/Put/seqid=0 2024-11-11T18:14:06,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742028_1204 (size=31105) 2024-11-11T18:14:06,462 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5c7af1c27173434c8493cac5bd8458a5 2024-11-11T18:14:06,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/5cc99770e9c54e9d9ea6ec2931c32f22 is 50, key is test_row_0/B:col10/1731348845442/Put/seqid=0 2024-11-11T18:14:06,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742029_1205 (size=12151) 2024-11-11T18:14:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:06,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:06,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348906619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348906619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T18:14:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348906725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348906731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41618 deadline: 1731348906784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,786 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41574 deadline: 1731348906804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,807 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41614 deadline: 1731348906845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,848 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,892 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/5cc99770e9c54e9d9ea6ec2931c32f22 2024-11-11T18:14:06,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/5bc69a32e8f54ba8be4999d845e8f7ed is 50, key is test_row_0/C:col10/1731348845442/Put/seqid=0 2024-11-11T18:14:06,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742030_1206 (size=12151) 2024-11-11T18:14:06,927 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/5bc69a32e8f54ba8be4999d845e8f7ed 2024-11-11T18:14:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/5c7af1c27173434c8493cac5bd8458a5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5 2024-11-11T18:14:06,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348906935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348906944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:06,945 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5, entries=150, sequenceid=218, filesize=30.4 K 2024-11-11T18:14:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/5cc99770e9c54e9d9ea6ec2931c32f22 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5cc99770e9c54e9d9ea6ec2931c32f22 2024-11-11T18:14:06,954 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5cc99770e9c54e9d9ea6ec2931c32f22, entries=150, sequenceid=218, filesize=11.9 K 2024-11-11T18:14:06,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/5bc69a32e8f54ba8be4999d845e8f7ed as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5bc69a32e8f54ba8be4999d845e8f7ed 2024-11-11T18:14:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5bc69a32e8f54ba8be4999d845e8f7ed, entries=150, sequenceid=218, filesize=11.9 K 2024-11-11T18:14:06,977 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cc678935c00a295255833d490f0a10a8 in 588ms, sequenceid=218, compaction requested=false 2024-11-11T18:14:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:06,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T18:14:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T18:14:06,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T18:14:06,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 900 msec 2024-11-11T18:14:06,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 907 msec 2024-11-11T18:14:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T18:14:07,215 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:07,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T18:14:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T18:14:07,218 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:07,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:07,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:07,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T18:14:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:07,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:07,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:07,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:07,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:07,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f4fda6849aca49718f9d8199f061cbf2_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348846617/Put/seqid=0 2024-11-11T18:14:07,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742031_1207 (size=12304) 2024-11-11T18:14:07,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348907305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348907306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T18:14:07,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348907416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348907424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:07,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T18:14:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348907624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348907634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:07,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:07,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:07,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f4fda6849aca49718f9d8199f061cbf2_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4fda6849aca49718f9d8199f061cbf2_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:07,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/eec7c264ec37416695aeeb6d2f9f4d43, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:07,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/eec7c264ec37416695aeeb6d2f9f4d43 is 175, key is test_row_0/A:col10/1731348846617/Put/seqid=0 2024-11-11T18:14:07,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742032_1208 (size=31105) 2024-11-11T18:14:07,714 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/eec7c264ec37416695aeeb6d2f9f4d43 2024-11-11T18:14:07,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/dd2c2fbcb8d64829b026d587d70e2bd7 is 50, key is test_row_0/B:col10/1731348846617/Put/seqid=0 2024-11-11T18:14:07,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742033_1209 (size=12151) 2024-11-11T18:14:07,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T18:14:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348907935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348907945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:07,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:07,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:07,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:07,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:07,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:08,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:08,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:08,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:08,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:08,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/dd2c2fbcb8d64829b026d587d70e2bd7 2024-11-11T18:14:08,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7275c51ccd424b03a22eab42bd3a8d71 is 50, key is test_row_0/C:col10/1731348846617/Put/seqid=0 2024-11-11T18:14:08,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742034_1210 (size=12151) 2024-11-11T18:14:08,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7275c51ccd424b03a22eab42bd3a8d71 2024-11-11T18:14:08,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/eec7c264ec37416695aeeb6d2f9f4d43 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43 2024-11-11T18:14:08,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43, entries=150, sequenceid=233, filesize=30.4 K 2024-11-11T18:14:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/dd2c2fbcb8d64829b026d587d70e2bd7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dd2c2fbcb8d64829b026d587d70e2bd7 2024-11-11T18:14:08,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dd2c2fbcb8d64829b026d587d70e2bd7, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T18:14:08,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7275c51ccd424b03a22eab42bd3a8d71 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7275c51ccd424b03a22eab42bd3a8d71 2024-11-11T18:14:08,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7275c51ccd424b03a22eab42bd3a8d71, entries=150, sequenceid=233, filesize=11.9 K 2024-11-11T18:14:08,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cc678935c00a295255833d490f0a10a8 in 1009ms, sequenceid=233, compaction requested=true 2024-11-11T18:14:08,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:08,257 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:08,257 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:08,258 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:08,258 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:08,258 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,258 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8b8d577c0b5849df98373dd1ea162dcd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=91.5 K 2024-11-11T18:14:08,258 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,258 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8b8d577c0b5849df98373dd1ea162dcd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43] 2024-11-11T18:14:08,259 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:08,259 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:08,259 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,259 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/36b3c5ee684b43759cd628f5a055c5c3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5cc99770e9c54e9d9ea6ec2931c32f22, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dd2c2fbcb8d64829b026d587d70e2bd7] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=36.0 K 2024-11-11T18:14:08,259 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8d577c0b5849df98373dd1ea162dcd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731348844211 2024-11-11T18:14:08,260 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b3c5ee684b43759cd628f5a055c5c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731348844211 2024-11-11T18:14:08,260 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c7af1c27173434c8493cac5bd8458a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731348845404 2024-11-11T18:14:08,260 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cc99770e9c54e9d9ea6ec2931c32f22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731348845404 2024-11-11T18:14:08,260 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec7c264ec37416695aeeb6d2f9f4d43, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731348846608 2024-11-11T18:14:08,261 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting dd2c2fbcb8d64829b026d587d70e2bd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731348846608 2024-11-11T18:14:08,272 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:08,276 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:08,277 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/ea2d127699f848c3b9fb8a8ee19f1540 is 50, key is test_row_0/B:col10/1731348846617/Put/seqid=0 2024-11-11T18:14:08,277 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ebe86fff074c44c0a336050f1ebe39cb_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:08,279 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ebe86fff074c44c0a336050f1ebe39cb_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:08,279 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ebe86fff074c44c0a336050f1ebe39cb_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:08,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T18:14:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,306 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T18:14:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:08,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742036_1212 (size=4469) 2024-11-11T18:14:08,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742035_1211 (size=12629) 2024-11-11T18:14:08,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111231f95a546994945bd85a0b29d6f2364_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348847296/Put/seqid=0 2024-11-11T18:14:08,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742037_1213 (size=12304) 2024-11-11T18:14:08,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:08,361 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111231f95a546994945bd85a0b29d6f2364_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111231f95a546994945bd85a0b29d6f2364_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:08,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e6e7e00e439e4c519b5817ba38139da4, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:08,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e6e7e00e439e4c519b5817ba38139da4 is 175, key is test_row_0/A:col10/1731348847296/Put/seqid=0 2024-11-11T18:14:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T18:14:08,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742038_1214 (size=31105) 2024-11-11T18:14:08,370 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e6e7e00e439e4c519b5817ba38139da4 2024-11-11T18:14:08,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/911e26869d8a407bae3dd44ada9a861a is 50, key is test_row_0/B:col10/1731348847296/Put/seqid=0 2024-11-11T18:14:08,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742039_1215 (size=12151) 2024-11-11T18:14:08,390 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/911e26869d8a407bae3dd44ada9a861a 2024-11-11T18:14:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/5fdb7f2d5d05498f8ba54a2950cb92b7 is 50, key is test_row_0/C:col10/1731348847296/Put/seqid=0 2024-11-11T18:14:08,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742040_1216 (size=12151) 2024-11-11T18:14:08,447 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/5fdb7f2d5d05498f8ba54a2950cb92b7 2024-11-11T18:14:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:08,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:08,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e6e7e00e439e4c519b5817ba38139da4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4 2024-11-11T18:14:08,458 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4, entries=150, sequenceid=252, filesize=30.4 K 2024-11-11T18:14:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/911e26869d8a407bae3dd44ada9a861a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/911e26869d8a407bae3dd44ada9a861a 2024-11-11T18:14:08,465 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/911e26869d8a407bae3dd44ada9a861a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-11T18:14:08,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/5fdb7f2d5d05498f8ba54a2950cb92b7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5fdb7f2d5d05498f8ba54a2950cb92b7 2024-11-11T18:14:08,471 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5fdb7f2d5d05498f8ba54a2950cb92b7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-11T18:14:08,472 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=40.25 KB/41220 for cc678935c00a295255833d490f0a10a8 in 166ms, sequenceid=252, compaction requested=true 2024-11-11T18:14:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T18:14:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T18:14:08,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T18:14:08,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2530 sec 2024-11-11T18:14:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:08,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:08,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.2600 sec 2024-11-11T18:14:08,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:08,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:08,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:08,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:08,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:08,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:08,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e05e6118236b44ebab6ccd02b0b8be5f_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348848476/Put/seqid=0 2024-11-11T18:14:08,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742041_1217 (size=17484) 2024-11-11T18:14:08,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348908531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:08,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731348848483, max timestamp=1731348848536Num uniques 859; 2024-11-11T18:14:08,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T18:14:08,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731348848483, max timestamp=1731348848536Num uniques 859; 2024-11-11T18:14:08,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T18:14:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:08,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348848522, max timestamp=1731348848536Num uniques 859; 2024-11-11T18:14:08,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:08,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348908629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:08,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348908644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:08,730 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#181 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:08,731 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/58d988f6ad5b41f49295b555ee2c8788 is 175, key is test_row_0/A:col10/1731348846617/Put/seqid=0 2024-11-11T18:14:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348908734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742042_1218 (size=31583) 2024-11-11T18:14:08,742 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/ea2d127699f848c3b9fb8a8ee19f1540 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ea2d127699f848c3b9fb8a8ee19f1540 2024-11-11T18:14:08,751 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into ea2d127699f848c3b9fb8a8ee19f1540(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:08,751 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:08,751 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=13, startTime=1731348848257; duration=0sec 2024-11-11T18:14:08,751 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:08,751 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:08,751 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:08,757 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:08,758 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:14:08,758 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:08,758 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/3b9190fe6b06485e81b59d2bad69bc77, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5bc69a32e8f54ba8be4999d845e8f7ed, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7275c51ccd424b03a22eab42bd3a8d71, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5fdb7f2d5d05498f8ba54a2950cb92b7] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=47.8 K 2024-11-11T18:14:08,758 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b9190fe6b06485e81b59d2bad69bc77, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731348844211 2024-11-11T18:14:08,758 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc69a32e8f54ba8be4999d845e8f7ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731348845404 2024-11-11T18:14:08,761 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7275c51ccd424b03a22eab42bd3a8d71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731348846608 2024-11-11T18:14:08,761 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fdb7f2d5d05498f8ba54a2950cb92b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731348847296 2024-11-11T18:14:08,773 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#187 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:08,774 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/b028b93da5a2409d8975067825dca1c0 is 50, key is test_row_0/C:col10/1731348847296/Put/seqid=0 2024-11-11T18:14:08,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742043_1219 (size=12663) 2024-11-11T18:14:08,799 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/b028b93da5a2409d8975067825dca1c0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b028b93da5a2409d8975067825dca1c0 2024-11-11T18:14:08,805 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into b028b93da5a2409d8975067825dca1c0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:08,805 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:08,805 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=12, startTime=1731348848257; duration=0sec 2024-11-11T18:14:08,805 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:08,805 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:08,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348908855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:08,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:08,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e05e6118236b44ebab6ccd02b0b8be5f_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e05e6118236b44ebab6ccd02b0b8be5f_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:08,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e788f301c9b1455289054e13e98d3f46, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:08,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e788f301c9b1455289054e13e98d3f46 is 175, key is test_row_0/A:col10/1731348848476/Put/seqid=0 2024-11-11T18:14:08,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348908945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:08,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:08,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742044_1220 (size=48589) 2024-11-11T18:14:09,148 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/58d988f6ad5b41f49295b555ee2c8788 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/58d988f6ad5b41f49295b555ee2c8788 2024-11-11T18:14:09,159 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into 58d988f6ad5b41f49295b555ee2c8788(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:09,159 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:09,159 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=13, startTime=1731348848257; duration=0sec 2024-11-11T18:14:09,160 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:09,160 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348909165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:09,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348909265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:09,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e788f301c9b1455289054e13e98d3f46 2024-11-11T18:14:09,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/c4f4c26677a54084948851bc6cb9960b is 50, key is test_row_0/B:col10/1731348848476/Put/seqid=0 2024-11-11T18:14:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T18:14:09,389 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:09,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T18:14:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T18:14:09,392 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:09,393 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:09,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:09,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742045_1221 (size=12251) 2024-11-11T18:14:09,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/c4f4c26677a54084948851bc6cb9960b 2024-11-11T18:14:09,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/ff5db462cfdb4dfb8fec2df31ca1f756 is 50, key is test_row_0/C:col10/1731348848476/Put/seqid=0 2024-11-11T18:14:09,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742046_1222 (size=12251) 2024-11-11T18:14:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T18:14:09,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T18:14:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348909685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:09,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T18:14:09,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T18:14:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348909774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:09,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:09,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T18:14:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:09,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/ff5db462cfdb4dfb8fec2df31ca1f756 2024-11-11T18:14:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/e788f301c9b1455289054e13e98d3f46 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46 2024-11-11T18:14:09,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46, entries=250, sequenceid=263, filesize=47.5 K 2024-11-11T18:14:09,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/c4f4c26677a54084948851bc6cb9960b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c4f4c26677a54084948851bc6cb9960b 2024-11-11T18:14:09,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c4f4c26677a54084948851bc6cb9960b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T18:14:09,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/ff5db462cfdb4dfb8fec2df31ca1f756 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ff5db462cfdb4dfb8fec2df31ca1f756 2024-11-11T18:14:09,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ff5db462cfdb4dfb8fec2df31ca1f756, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T18:14:09,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cc678935c00a295255833d490f0a10a8 in 1438ms, sequenceid=263, compaction requested=true 2024-11-11T18:14:09,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:09,915 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:09,917 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:09,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:09,925 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:09,925 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:09,925 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,925 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ea2d127699f848c3b9fb8a8ee19f1540, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/911e26869d8a407bae3dd44ada9a861a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c4f4c26677a54084948851bc6cb9960b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=36.2 K 2024-11-11T18:14:09,926 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:09,926 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:09,926 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,926 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/58d988f6ad5b41f49295b555ee2c8788, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=108.7 K 2024-11-11T18:14:09,926 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:09,926 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/58d988f6ad5b41f49295b555ee2c8788, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46] 2024-11-11T18:14:09,926 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ea2d127699f848c3b9fb8a8ee19f1540, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731348846608 2024-11-11T18:14:09,927 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58d988f6ad5b41f49295b555ee2c8788, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731348846608 2024-11-11T18:14:09,927 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 911e26869d8a407bae3dd44ada9a861a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731348847296 2024-11-11T18:14:09,928 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f4c26677a54084948851bc6cb9960b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731348848450 2024-11-11T18:14:09,928 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e7e00e439e4c519b5817ba38139da4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731348847296 2024-11-11T18:14:09,928 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e788f301c9b1455289054e13e98d3f46, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731348848450 2024-11-11T18:14:09,947 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:09,948 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f6ce4d710d4e4862be62d12104c01e96 is 50, key is test_row_0/B:col10/1731348848476/Put/seqid=0 2024-11-11T18:14:09,949 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:09,953 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113f0ebd48da9b473191d8c8a2001d1339_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:09,955 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113f0ebd48da9b473191d8c8a2001d1339_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:09,955 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113f0ebd48da9b473191d8c8a2001d1339_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:09,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742047_1223 (size=12831) 2024-11-11T18:14:09,984 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f6ce4d710d4e4862be62d12104c01e96 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f6ce4d710d4e4862be62d12104c01e96 2024-11-11T18:14:09,992 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into f6ce4d710d4e4862be62d12104c01e96(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:09,993 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:09,993 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=13, startTime=1731348849917; duration=0sec 2024-11-11T18:14:09,993 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:09,993 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:09,993 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T18:14:09,994 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T18:14:09,994 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T18:14:09,994 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. because compaction request was cancelled 2024-11-11T18:14:09,994 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:09,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742048_1224 (size=4469) 2024-11-11T18:14:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T18:14:10,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T18:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:10,016 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-11T18:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:10,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:10,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118b3ead4ec08b4fb9a3bb2eafb0e4bcd1_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348848617/Put/seqid=0 2024-11-11T18:14:10,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742049_1225 (size=12454) 2024-11-11T18:14:10,401 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#191 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:10,402 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/923eedfce4b648e789dc793146d2db64 is 175, key is test_row_0/A:col10/1731348848476/Put/seqid=0 2024-11-11T18:14:10,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742050_1226 (size=31785) 2024-11-11T18:14:10,419 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/923eedfce4b648e789dc793146d2db64 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/923eedfce4b648e789dc793146d2db64 2024-11-11T18:14:10,439 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into 923eedfce4b648e789dc793146d2db64(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:10,439 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:10,439 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=13, startTime=1731348849915; duration=0sec 2024-11-11T18:14:10,439 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:10,439 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:10,454 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118b3ead4ec08b4fb9a3bb2eafb0e4bcd1_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b3ead4ec08b4fb9a3bb2eafb0e4bcd1_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bc25198deedd47f3bdefe9fb28e4b631, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:10,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bc25198deedd47f3bdefe9fb28e4b631 is 175, key is test_row_0/A:col10/1731348848617/Put/seqid=0 2024-11-11T18:14:10,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742051_1227 (size=31255) 2024-11-11T18:14:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T18:14:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:10,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:10,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348910752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:10,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:10,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348910789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:10,792 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:10,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348910864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:10,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:10,892 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bc25198deedd47f3bdefe9fb28e4b631 2024-11-11T18:14:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/92d6c1a9daf84808a5c9342a57750f95 is 50, key is test_row_0/B:col10/1731348848617/Put/seqid=0 2024-11-11T18:14:10,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742052_1228 (size=12301) 2024-11-11T18:14:11,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348911075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:11,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:11,341 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/92d6c1a9daf84808a5c9342a57750f95 2024-11-11T18:14:11,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/64e8b121aefa487f834dccc9f81c255f is 50, key is test_row_0/C:col10/1731348848617/Put/seqid=0 2024-11-11T18:14:11,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348911395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742053_1229 (size=12301) 2024-11-11T18:14:11,410 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/64e8b121aefa487f834dccc9f81c255f 2024-11-11T18:14:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bc25198deedd47f3bdefe9fb28e4b631 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631 2024-11-11T18:14:11,422 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631, entries=150, sequenceid=301, filesize=30.5 K 2024-11-11T18:14:11,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/92d6c1a9daf84808a5c9342a57750f95 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/92d6c1a9daf84808a5c9342a57750f95 2024-11-11T18:14:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,431 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/92d6c1a9daf84808a5c9342a57750f95, entries=150, sequenceid=301, filesize=12.0 K 2024-11-11T18:14:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/64e8b121aefa487f834dccc9f81c255f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/64e8b121aefa487f834dccc9f81c255f 2024-11-11T18:14:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,448 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/64e8b121aefa487f834dccc9f81c255f, entries=150, sequenceid=301, filesize=12.0 K 2024-11-11T18:14:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,449 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for cc678935c00a295255833d490f0a10a8 in 1433ms, sequenceid=301, compaction requested=true 2024-11-11T18:14:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:11,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T18:14:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T18:14:11,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T18:14:11,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0580 sec 2024-11-11T18:14:11,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0660 sec 2024-11-11T18:14:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T18:14:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,537 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:11,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-11T18:14:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T18:14:11,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:11,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T18:14:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T18:14:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:11,698 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:11,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:11,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:11,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:11,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:11,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bc8a0f07fbf242b381b2ffc1eb21f3a0_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348850737/Put/seqid=0 2024-11-11T18:14:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742054_1230 (size=9914) 2024-11-11T18:14:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,764 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bc8a0f07fbf242b381b2ffc1eb21f3a0_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bc8a0f07fbf242b381b2ffc1eb21f3a0_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/46f6a34af32342cc9dc8e9272588fe6e, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/46f6a34af32342cc9dc8e9272588fe6e is 175, key is test_row_0/A:col10/1731348850737/Put/seqid=0 2024-11-11T18:14:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742055_1231 (size=22561) 2024-11-11T18:14:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,801 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/46f6a34af32342cc9dc8e9272588fe6e 2024-11-11T18:14:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/42a9a667dfbe4b609ecee07700366979 is 50, key is test_row_0/B:col10/1731348850737/Put/seqid=0 2024-11-11T18:14:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T18:14:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742056_1232 (size=9857) 2024-11-11T18:14:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:12,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348852007, max timestamp=1731348852044Num uniques 859; 2024-11-11T18:14:12,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:12,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348852007, max timestamp=1731348852044Num uniques 859; 2024-11-11T18:14:12,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:12,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348852007, max timestamp=1731348852044Num uniques 859; 2024-11-11T18:14:12,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348912146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T18:14:12,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348912254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:12,261 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/42a9a667dfbe4b609ecee07700366979 2024-11-11T18:14:12,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/97be8961a67c4044aebfb61a6b7b3b82 is 50, key is test_row_0/C:col10/1731348850737/Put/seqid=0 2024-11-11T18:14:12,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742057_1233 (size=9857) 2024-11-11T18:14:12,285 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/97be8961a67c4044aebfb61a6b7b3b82 2024-11-11T18:14:12,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/46f6a34af32342cc9dc8e9272588fe6e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e 2024-11-11T18:14:12,300 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e, entries=100, sequenceid=313, filesize=22.0 K 2024-11-11T18:14:12,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/42a9a667dfbe4b609ecee07700366979 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/42a9a667dfbe4b609ecee07700366979 2024-11-11T18:14:12,306 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/42a9a667dfbe4b609ecee07700366979, entries=100, sequenceid=313, filesize=9.6 K 2024-11-11T18:14:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/97be8961a67c4044aebfb61a6b7b3b82 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/97be8961a67c4044aebfb61a6b7b3b82 2024-11-11T18:14:12,319 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/97be8961a67c4044aebfb61a6b7b3b82, entries=100, sequenceid=313, filesize=9.6 K 2024-11-11T18:14:12,320 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cc678935c00a295255833d490f0a10a8 in 622ms, sequenceid=313, compaction requested=true 2024-11-11T18:14:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-11T18:14:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-11T18:14:12,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T18:14:12,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 778 msec 2024-11-11T18:14:12,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 787 msec 2024-11-11T18:14:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:12,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T18:14:12,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:12,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:12,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:12,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:12,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:12,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:12,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a3df66840c6495da1d822cfc7fbccb0_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:12,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742058_1234 (size=14994) 2024-11-11T18:14:12,494 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:12,499 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111a3df66840c6495da1d822cfc7fbccb0_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a3df66840c6495da1d822cfc7fbccb0_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:12,500 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/61ba5bf092fd4487bf7812afdaf00900, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:12,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/61ba5bf092fd4487bf7812afdaf00900 is 175, key is test_row_0/A:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:12,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742059_1235 (size=39949) 2024-11-11T18:14:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348912510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:12,517 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/61ba5bf092fd4487bf7812afdaf00900 2024-11-11T18:14:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:12,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/3c36de2f28eb4b748b477da00294d441 is 50, key is test_row_0/B:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:12,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742060_1236 (size=12301) 2024-11-11T18:14:12,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/3c36de2f28eb4b748b477da00294d441 2024-11-11T18:14:12,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/baa85b0588eb45aba5bfbbf8ee9b70e6 is 50, key is test_row_0/C:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:12,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742061_1237 (size=12301) 2024-11-11T18:14:12,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348912624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T18:14:12,674 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:12,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T18:14:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T18:14:12,677 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:12,678 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:12,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T18:14:12,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1731348912804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:12,807 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in cc678935c00a295255833d490f0a10a8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:12,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T18:14:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:12,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:12,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348912835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:12,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/baa85b0588eb45aba5bfbbf8ee9b70e6 2024-11-11T18:14:12,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/61ba5bf092fd4487bf7812afdaf00900 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900 2024-11-11T18:14:12,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900, entries=200, sequenceid=350, filesize=39.0 K 2024-11-11T18:14:12,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/3c36de2f28eb4b748b477da00294d441 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/3c36de2f28eb4b748b477da00294d441 2024-11-11T18:14:12,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/3c36de2f28eb4b748b477da00294d441, entries=150, sequenceid=350, filesize=12.0 K 2024-11-11T18:14:12,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T18:14:12,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:12,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:12,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:12,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:12,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:12,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/baa85b0588eb45aba5bfbbf8ee9b70e6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/baa85b0588eb45aba5bfbbf8ee9b70e6 2024-11-11T18:14:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T18:14:12,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/baa85b0588eb45aba5bfbbf8ee9b70e6, entries=150, sequenceid=350, filesize=12.0 K 2024-11-11T18:14:12,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cc678935c00a295255833d490f0a10a8 in 528ms, sequenceid=350, compaction requested=true 2024-11-11T18:14:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:12,998 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:12,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:12,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:12,999 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:12,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:13,000 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125550 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:13,000 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:13,000 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,000 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/923eedfce4b648e789dc793146d2db64, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=122.6 K 2024-11-11T18:14:13,000 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,000 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/923eedfce4b648e789dc793146d2db64, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900] 2024-11-11T18:14:13,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:13,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:13,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:13,001 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923eedfce4b648e789dc793146d2db64, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731348848450 2024-11-11T18:14:13,001 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc25198deedd47f3bdefe9fb28e4b631, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731348848579 2024-11-11T18:14:13,001 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:13,002 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:13,002 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,002 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f6ce4d710d4e4862be62d12104c01e96, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/92d6c1a9daf84808a5c9342a57750f95, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/42a9a667dfbe4b609ecee07700366979, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/3c36de2f28eb4b748b477da00294d441] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=46.2 K 2024-11-11T18:14:13,002 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f6a34af32342cc9dc8e9272588fe6e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731348850737 2024-11-11T18:14:13,002 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ce4d710d4e4862be62d12104c01e96, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731348848450 2024-11-11T18:14:13,002 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d6c1a9daf84808a5c9342a57750f95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731348848579 2024-11-11T18:14:13,002 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ba5bf092fd4487bf7812afdaf00900, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731348852124 2024-11-11T18:14:13,003 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a9a667dfbe4b609ecee07700366979, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731348850737 2024-11-11T18:14:13,003 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c36de2f28eb4b748b477da00294d441, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731348852127 2024-11-11T18:14:13,027 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:13,027 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/aab9985b610a4eac8a4f8ecf73ab0c00 is 50, key is test_row_0/B:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:13,034 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,053 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111138a6d251a134df6bfb7243e486bbe93_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,056 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111138a6d251a134df6bfb7243e486bbe93_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,056 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111138a6d251a134df6bfb7243e486bbe93_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742062_1238 (size=13017) 2024-11-11T18:14:13,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742063_1239 (size=4469) 2024-11-11T18:14:13,090 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#202 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:13,090 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/528c3dadb88647dfb708054019e886bb is 175, key is test_row_0/A:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:13,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742064_1240 (size=31971) 2024-11-11T18:14:13,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T18:14:13,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,144 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:13,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:13,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111142b8ec299c644c9092b2847ca6e454a4_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348852493/Put/seqid=0 2024-11-11T18:14:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742065_1241 (size=12454) 2024-11-11T18:14:13,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:13,174 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111142b8ec299c644c9092b2847ca6e454a4_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111142b8ec299c644c9092b2847ca6e454a4_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/d126ee1f03e043269540a0ca857f57f2, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/d126ee1f03e043269540a0ca857f57f2 is 175, key is test_row_0/A:col10/1731348852493/Put/seqid=0 2024-11-11T18:14:13,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742066_1242 (size=31255) 2024-11-11T18:14:13,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348853221, max timestamp=1731348853265Num uniques 859; 2024-11-11T18:14:13,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348853221, max timestamp=1731348853265Num uniques 859; 2024-11-11T18:14:13,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:13,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348853221, max timestamp=1731348853265Num uniques 859; 2024-11-11T18:14:13,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:13,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T18:14:13,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348913408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:13,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:13,470 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/aab9985b610a4eac8a4f8ecf73ab0c00 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/aab9985b610a4eac8a4f8ecf73ab0c00 2024-11-11T18:14:13,487 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into aab9985b610a4eac8a4f8ecf73ab0c00(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:13,487 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:13,487 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=12, startTime=1731348852999; duration=0sec 2024-11-11T18:14:13,488 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:13,488 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:13,488 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T18:14:13,490 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59373 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T18:14:13,490 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:14:13,491 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,491 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b028b93da5a2409d8975067825dca1c0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ff5db462cfdb4dfb8fec2df31ca1f756, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/64e8b121aefa487f834dccc9f81c255f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/97be8961a67c4044aebfb61a6b7b3b82, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/baa85b0588eb45aba5bfbbf8ee9b70e6] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=58.0 K 2024-11-11T18:14:13,493 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b028b93da5a2409d8975067825dca1c0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731348847296 2024-11-11T18:14:13,494 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ff5db462cfdb4dfb8fec2df31ca1f756, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731348848450 2024-11-11T18:14:13,494 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e8b121aefa487f834dccc9f81c255f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731348848579 2024-11-11T18:14:13,495 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 97be8961a67c4044aebfb61a6b7b3b82, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731348850737 2024-11-11T18:14:13,495 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting baa85b0588eb45aba5bfbbf8ee9b70e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731348852127 2024-11-11T18:14:13,519 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/528c3dadb88647dfb708054019e886bb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/528c3dadb88647dfb708054019e886bb 2024-11-11T18:14:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348913514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:13,527 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into 528c3dadb88647dfb708054019e886bb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:13,527 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:13,527 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=12, startTime=1731348852998; duration=0sec 2024-11-11T18:14:13,527 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:13,527 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:13,532 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:13,532 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/50c7c9ab66674ecf921c07d2c585032f is 50, key is test_row_0/C:col10/1731348852468/Put/seqid=0 2024-11-11T18:14:13,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742067_1243 (size=12983) 2024-11-11T18:14:13,565 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/50c7c9ab66674ecf921c07d2c585032f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/50c7c9ab66674ecf921c07d2c585032f 2024-11-11T18:14:13,570 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into 50c7c9ab66674ecf921c07d2c585032f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:13,570 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:13,571 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=11, startTime=1731348853001; duration=0sec 2024-11-11T18:14:13,571 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:13,571 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:13,583 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/d126ee1f03e043269540a0ca857f57f2 2024-11-11T18:14:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/4d15f044e6314ff7a6a2c5bbea51cdc9 is 50, key is test_row_0/B:col10/1731348852493/Put/seqid=0 2024-11-11T18:14:13,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742068_1244 (size=12301) 2024-11-11T18:14:13,642 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/4d15f044e6314ff7a6a2c5bbea51cdc9 2024-11-11T18:14:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/2f358f2f121c4d93b05347d3cd063535 is 50, key is test_row_0/C:col10/1731348852493/Put/seqid=0 2024-11-11T18:14:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742069_1245 (size=12301) 2024-11-11T18:14:13,714 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/2f358f2f121c4d93b05347d3cd063535 2024-11-11T18:14:13,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/d126ee1f03e043269540a0ca857f57f2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2 2024-11-11T18:14:13,725 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2, entries=150, sequenceid=360, filesize=30.5 K 2024-11-11T18:14:13,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/4d15f044e6314ff7a6a2c5bbea51cdc9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4d15f044e6314ff7a6a2c5bbea51cdc9 2024-11-11T18:14:13,734 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4d15f044e6314ff7a6a2c5bbea51cdc9, entries=150, sequenceid=360, filesize=12.0 K 2024-11-11T18:14:13,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/2f358f2f121c4d93b05347d3cd063535 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2f358f2f121c4d93b05347d3cd063535 2024-11-11T18:14:13,740 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2f358f2f121c4d93b05347d3cd063535, entries=150, sequenceid=360, filesize=12.0 K 2024-11-11T18:14:13,741 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cc678935c00a295255833d490f0a10a8 in 597ms, sequenceid=360, compaction requested=false 2024-11-11T18:14:13,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:13,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T18:14:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T18:14:13,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348853345, max timestamp=1731348853744Num uniques 1018; 2024-11-11T18:14:13,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:14:13,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T18:14:13,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0650 sec 2024-11-11T18:14:13,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.0740 sec 2024-11-11T18:14:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348853345, max timestamp=1731348853744Num uniques 1018; 2024-11-11T18:14:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:14:13,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348853345, max timestamp=1731348853744Num uniques 1018; 2024-11-11T18:14:13,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:14:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:13,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-11T18:14:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:13,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:13,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b4d6d9d3aad74347b73eeb57149c49fb_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T18:14:13,815 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:13,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T18:14:13,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742070_1246 (size=17534) 2024-11-11T18:14:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T18:14:13,818 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:13,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:13,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:13,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:13,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b4d6d9d3aad74347b73eeb57149c49fb_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b4d6d9d3aad74347b73eeb57149c49fb_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:13,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/8d5028253a5748b581dec9d9429bffb8, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/8d5028253a5748b581dec9d9429bffb8 is 175, key is test_row_0/A:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:13,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742071_1247 (size=48639) 2024-11-11T18:14:13,862 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/8d5028253a5748b581dec9d9429bffb8 2024-11-11T18:14:13,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/623e746f97294f93b3a31d907da9cbd6 is 50, key is test_row_0/B:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:13,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348853839, max timestamp=1731348853891Num uniques 859; 2024-11-11T18:14:13,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:13,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348853839, max timestamp=1731348853891Num uniques 859; 2024-11-11T18:14:13,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:13,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348853839, max timestamp=1731348853891Num uniques 859; 2024-11-11T18:14:13,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:13,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742072_1248 (size=12301) 2024-11-11T18:14:13,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/623e746f97294f93b3a31d907da9cbd6 2024-11-11T18:14:13,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/493c54bb121b47769497373300798ab2 is 50, key is test_row_0/C:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T18:14:13,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742073_1249 (size=12301) 2024-11-11T18:14:13,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/493c54bb121b47769497373300798ab2 2024-11-11T18:14:13,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/8d5028253a5748b581dec9d9429bffb8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8 2024-11-11T18:14:13,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8, entries=250, sequenceid=405, filesize=47.5 K 2024-11-11T18:14:13,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/623e746f97294f93b3a31d907da9cbd6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/623e746f97294f93b3a31d907da9cbd6 2024-11-11T18:14:13,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/623e746f97294f93b3a31d907da9cbd6, entries=150, sequenceid=405, filesize=12.0 K 2024-11-11T18:14:13,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/493c54bb121b47769497373300798ab2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/493c54bb121b47769497373300798ab2 2024-11-11T18:14:13,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/493c54bb121b47769497373300798ab2, entries=150, sequenceid=405, filesize=12.0 K 2024-11-11T18:14:13,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=80.51 KB/82440 for cc678935c00a295255833d490f0a10a8 in 197ms, sequenceid=405, compaction requested=true 2024-11-11T18:14:13,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:13,967 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:13,968 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:13,968 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:13,968 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,968 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/528c3dadb88647dfb708054019e886bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=109.2 K 2024-11-11T18:14:13,968 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,969 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/528c3dadb88647dfb708054019e886bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8] 2024-11-11T18:14:13,969 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528c3dadb88647dfb708054019e886bb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731348852127 2024-11-11T18:14:13,970 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d126ee1f03e043269540a0ca857f57f2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731348852484 2024-11-11T18:14:13,970 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d5028253a5748b581dec9d9429bffb8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731348853345 2024-11-11T18:14:13,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T18:14:13,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,972 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=80.51 KB heapSize=220.92 KB 2024-11-11T18:14:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:13,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:13,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:13,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:13,979 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:13,981 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:13,985 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:13,985 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:13,985 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:13,986 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/aab9985b610a4eac8a4f8ecf73ab0c00, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4d15f044e6314ff7a6a2c5bbea51cdc9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/623e746f97294f93b3a31d907da9cbd6] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=36.7 K 2024-11-11T18:14:13,987 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting aab9985b610a4eac8a4f8ecf73ab0c00, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731348852127 2024-11-11T18:14:13,987 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d15f044e6314ff7a6a2c5bbea51cdc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731348852484 2024-11-11T18:14:13,988 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 623e746f97294f93b3a31d907da9cbd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731348853345 2024-11-11T18:14:14,001 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111307a6c3ce5fa4cfbac8deb1b7604d9ff_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:14,003 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111307a6c3ce5fa4cfbac8deb1b7604d9ff_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:14,003 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111307a6c3ce5fa4cfbac8deb1b7604d9ff_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:14,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111123d9745d18ed45969d7db61cdeb60da0_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348853943/Put/seqid=0 2024-11-11T18:14:14,026 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:14,027 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f861caab6ffd46138d54a6ab26e0e706 is 50, key is test_row_0/B:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:14,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742074_1250 (size=4469) 2024-11-11T18:14:14,071 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#210 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:14,072 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/ccefdefb520c494f83ae6983abe947e5 is 175, key is test_row_0/A:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:14,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742075_1251 (size=14994) 2024-11-11T18:14:14,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742076_1252 (size=13119) 2024-11-11T18:14:14,111 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/f861caab6ffd46138d54a6ab26e0e706 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f861caab6ffd46138d54a6ab26e0e706 2024-11-11T18:14:14,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742077_1253 (size=32073) 2024-11-11T18:14:14,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348914106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:14,119 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into f861caab6ffd46138d54a6ab26e0e706(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:14,119 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:14,119 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=13, startTime=1731348853979; duration=0sec 2024-11-11T18:14:14,119 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:14,119 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:14,119 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:14,120 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:14,120 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:14:14,121 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:14,121 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/50c7c9ab66674ecf921c07d2c585032f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2f358f2f121c4d93b05347d3cd063535, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/493c54bb121b47769497373300798ab2] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=36.7 K 2024-11-11T18:14:14,121 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 50c7c9ab66674ecf921c07d2c585032f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731348852127 2024-11-11T18:14:14,129 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f358f2f121c4d93b05347d3cd063535, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731348852484 2024-11-11T18:14:14,133 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 493c54bb121b47769497373300798ab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731348853345 2024-11-11T18:14:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T18:14:14,139 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/ccefdefb520c494f83ae6983abe947e5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ccefdefb520c494f83ae6983abe947e5 2024-11-11T18:14:14,148 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into ccefdefb520c494f83ae6983abe947e5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:14,148 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:14,148 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=13, startTime=1731348853967; duration=0sec 2024-11-11T18:14:14,148 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:14,149 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:14,185 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:14,186 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/9d07015b754b41ac860cd5e24fb310ad is 50, key is test_row_0/C:col10/1731348853769/Put/seqid=0 2024-11-11T18:14:14,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742078_1254 (size=13085) 2024-11-11T18:14:14,223 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/9d07015b754b41ac860cd5e24fb310ad as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/9d07015b754b41ac860cd5e24fb310ad 2024-11-11T18:14:14,229 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into 9d07015b754b41ac860cd5e24fb310ad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:14,229 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:14,229 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=13, startTime=1731348853984; duration=0sec 2024-11-11T18:14:14,229 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:14,229 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:14,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348914225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:14,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T18:14:14,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348914444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:14,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:14,495 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111123d9745d18ed45969d7db61cdeb60da0_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111123d9745d18ed45969d7db61cdeb60da0_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/b193a10fe02c4c8a9a9d05b5fc70a784, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/b193a10fe02c4c8a9a9d05b5fc70a784 is 175, key is test_row_0/A:col10/1731348853943/Put/seqid=0 2024-11-11T18:14:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742079_1255 (size=39949) 2024-11-11T18:14:14,512 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/b193a10fe02c4c8a9a9d05b5fc70a784 2024-11-11T18:14:14,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/43d0c6f088ec44339be37a2815c3f5e9 is 50, key is test_row_0/B:col10/1731348853943/Put/seqid=0 2024-11-11T18:14:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742080_1256 (size=12301) 2024-11-11T18:14:14,532 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/43d0c6f088ec44339be37a2815c3f5e9 2024-11-11T18:14:14,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8ceb5e88ebe34a51a5310e6a0b39ab98 is 50, key is test_row_0/C:col10/1731348853943/Put/seqid=0 2024-11-11T18:14:14,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742081_1257 (size=12301) 2024-11-11T18:14:14,607 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8ceb5e88ebe34a51a5310e6a0b39ab98 2024-11-11T18:14:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/b193a10fe02c4c8a9a9d05b5fc70a784 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784 2024-11-11T18:14:14,620 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784, entries=200, sequenceid=434, filesize=39.0 K 2024-11-11T18:14:14,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/43d0c6f088ec44339be37a2815c3f5e9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/43d0c6f088ec44339be37a2815c3f5e9 2024-11-11T18:14:14,628 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/43d0c6f088ec44339be37a2815c3f5e9, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T18:14:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/8ceb5e88ebe34a51a5310e6a0b39ab98 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8ceb5e88ebe34a51a5310e6a0b39ab98 2024-11-11T18:14:14,645 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8ceb5e88ebe34a51a5310e6a0b39ab98, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T18:14:14,646 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.80 KB/246576, currentSize=93.93 KB/96180 for cc678935c00a295255833d490f0a10a8 in 675ms, sequenceid=434, compaction requested=false 2024-11-11T18:14:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:14,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T18:14:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T18:14:14,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T18:14:14,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 830 msec 2024-11-11T18:14:14,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 837 msec 2024-11-11T18:14:14,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T18:14:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:14,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:14,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111793a5ca3ca71409ca865c3cce7a210bb_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:14,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742082_1258 (size=14994) 2024-11-11T18:14:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348914810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:14,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348914924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T18:14:14,955 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:14,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T18:14:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:14,957 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:14,958 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:14,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:15,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:15,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:15,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348915135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:15,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111793a5ca3ca71409ca865c3cce7a210bb_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111793a5ca3ca71409ca865c3cce7a210bb_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:15,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/817eec6b3f8a476585bed82404fab2e8, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:15,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/817eec6b3f8a476585bed82404fab2e8 is 175, key is test_row_0/A:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742083_1259 (size=39949) 2024-11-11T18:14:15,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:15,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:15,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:15,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348915445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:15,560 INFO [Thread-760 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-760. 2024-11-11T18:14:15,560 DEBUG [Thread-760 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:15,560 DEBUG [Thread-760 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,560 DEBUG [Thread-760 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:15,564 INFO [Thread-758 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-758. 2024-11-11T18:14:15,564 DEBUG [Thread-758 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:15,564 DEBUG [Thread-758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,564 DEBUG [Thread-758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,564 INFO [Thread-762 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-762. 2024-11-11T18:14:15,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:15,564 DEBUG [Thread-762 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:15,564 DEBUG [Thread-762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,564 DEBUG [Thread-762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:15,565 INFO [Thread-764 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-764. 2024-11-11T18:14:15,565 DEBUG [Thread-764 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:15,565 DEBUG [Thread-764 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,565 DEBUG [Thread-764 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:15,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:15,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:15,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/817eec6b3f8a476585bed82404fab2e8 2024-11-11T18:14:15,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/0ae793c369a744a68fde7d2548e47d57 is 50, key is test_row_0/B:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:15,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742084_1260 (size=12301) 2024-11-11T18:14:15,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:15,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41620 deadline: 1731348915964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:16,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/0ae793c369a744a68fde7d2548e47d57 2024-11-11T18:14:16,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7df61ebb958944ce9d414629e4e0e2bf is 50, key is test_row_0/C:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742085_1261 (size=12301) 2024-11-11T18:14:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:16,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:16,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7df61ebb958944ce9d414629e4e0e2bf 2024-11-11T18:14:16,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/817eec6b3f8a476585bed82404fab2e8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8 2024-11-11T18:14:16,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8, entries=200, sequenceid=455, filesize=39.0 K 2024-11-11T18:14:16,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/0ae793c369a744a68fde7d2548e47d57 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/0ae793c369a744a68fde7d2548e47d57 2024-11-11T18:14:16,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/0ae793c369a744a68fde7d2548e47d57, entries=150, sequenceid=455, filesize=12.0 K 2024-11-11T18:14:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/7df61ebb958944ce9d414629e4e0e2bf as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7df61ebb958944ce9d414629e4e0e2bf 2024-11-11T18:14:16,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7df61ebb958944ce9d414629e4e0e2bf, entries=150, sequenceid=455, filesize=12.0 K 2024-11-11T18:14:16,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cc678935c00a295255833d490f0a10a8 in 1721ms, sequenceid=455, compaction requested=true 2024-11-11T18:14:16,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cc678935c00a295255833d490f0a10a8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:16,480 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:16,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:16,480 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:16,480 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:16,480 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:16,480 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/A is initiating minor compaction (all files) 2024-11-11T18:14:16,480 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/B is initiating minor compaction (all files) 2024-11-11T18:14:16,481 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/A in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,481 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/B in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,481 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ccefdefb520c494f83ae6983abe947e5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=109.3 K 2024-11-11T18:14:16,481 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f861caab6ffd46138d54a6ab26e0e706, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/43d0c6f088ec44339be37a2815c3f5e9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/0ae793c369a744a68fde7d2548e47d57] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=36.8 K 2024-11-11T18:14:16,481 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,481 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ccefdefb520c494f83ae6983abe947e5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8] 2024-11-11T18:14:16,481 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f861caab6ffd46138d54a6ab26e0e706, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731348853345 2024-11-11T18:14:16,481 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccefdefb520c494f83ae6983abe947e5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731348853345 2024-11-11T18:14:16,481 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b193a10fe02c4c8a9a9d05b5fc70a784, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731348853919 2024-11-11T18:14:16,481 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 43d0c6f088ec44339be37a2815c3f5e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731348853943 2024-11-11T18:14:16,482 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ae793c369a744a68fde7d2548e47d57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731348854099 2024-11-11T18:14:16,482 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 817eec6b3f8a476585bed82404fab2e8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731348854033 2024-11-11T18:14:16,489 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:16,490 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/990014b389f542e79b663323319fc75f is 50, key is test_row_0/B:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:16,491 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:16,493 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114e1e8b5f6bf04f27b4a7b45c437bda2d_cc678935c00a295255833d490f0a10a8 store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:16,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742086_1262 (size=13221) 2024-11-11T18:14:16,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T18:14:16,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,495 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T18:14:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:16,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:16,501 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/990014b389f542e79b663323319fc75f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/990014b389f542e79b663323319fc75f 2024-11-11T18:14:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bd7dd44c3ac8410dbc36aa82955e1be4_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348854808/Put/seqid=0 2024-11-11T18:14:16,506 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/B of cc678935c00a295255833d490f0a10a8 into 990014b389f542e79b663323319fc75f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:16,507 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:16,507 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/B, priority=13, startTime=1731348856480; duration=0sec 2024-11-11T18:14:16,507 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:16,507 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:B 2024-11-11T18:14:16,507 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:16,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742087_1263 (size=12454) 2024-11-11T18:14:16,508 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:16,508 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): cc678935c00a295255833d490f0a10a8/C is initiating minor compaction (all files) 2024-11-11T18:14:16,508 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cc678935c00a295255833d490f0a10a8/C in TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:16,508 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/9d07015b754b41ac860cd5e24fb310ad, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8ceb5e88ebe34a51a5310e6a0b39ab98, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7df61ebb958944ce9d414629e4e0e2bf] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp, totalSize=36.8 K 2024-11-11T18:14:16,509 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d07015b754b41ac860cd5e24fb310ad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731348853345 2024-11-11T18:14:16,510 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ceb5e88ebe34a51a5310e6a0b39ab98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731348853943 2024-11-11T18:14:16,510 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df61ebb958944ce9d414629e4e0e2bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1731348854099 2024-11-11T18:14:16,526 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#C#compaction#222 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:16,527 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/dca2c31cfb5d40f7b9494b4522440c40 is 50, key is test_row_0/C:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:16,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742088_1264 (size=13187) 2024-11-11T18:14:16,535 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114e1e8b5f6bf04f27b4a7b45c437bda2d_cc678935c00a295255833d490f0a10a8, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:16,536 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114e1e8b5f6bf04f27b4a7b45c437bda2d_cc678935c00a295255833d490f0a10a8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:16,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742089_1265 (size=4469) 2024-11-11T18:14:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:16,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. as already flushing 2024-11-11T18:14:16,817 INFO [Thread-753 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-753. 2024-11-11T18:14:16,817 DEBUG [Thread-753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:16,817 DEBUG [Thread-753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,817 DEBUG [Thread-753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:16,837 INFO [Thread-747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-747. 2024-11-11T18:14:16,837 DEBUG [Thread-747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:16,837 DEBUG [Thread-747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,837 DEBUG [Thread-747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:16,846 INFO [Thread-755 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-755. 2024-11-11T18:14:16,846 DEBUG [Thread-755 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:16,846 DEBUG [Thread-755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,846 DEBUG [Thread-755 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:16,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:16,913 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bd7dd44c3ac8410dbc36aa82955e1be4_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bd7dd44c3ac8410dbc36aa82955e1be4_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:16,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/fd4e49b7072a48dfa6a0e2f7752af17f, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:16,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/fd4e49b7072a48dfa6a0e2f7752af17f is 175, key is test_row_0/A:col10/1731348854808/Put/seqid=0 2024-11-11T18:14:16,926 INFO [Thread-749 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-749. 2024-11-11T18:14:16,926 DEBUG [Thread-749 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:16,926 DEBUG [Thread-749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,926 DEBUG [Thread-749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:16,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742090_1266 (size=31255) 2024-11-11T18:14:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=469, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/fd4e49b7072a48dfa6a0e2f7752af17f 2024-11-11T18:14:16,938 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/dca2c31cfb5d40f7b9494b4522440c40 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/dca2c31cfb5d40f7b9494b4522440c40 2024-11-11T18:14:16,943 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cc678935c00a295255833d490f0a10a8#A#compaction#220 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:16,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/eafe0b82fce742ad9b64c4b84da793f5 is 50, key is test_row_0/B:col10/1731348854808/Put/seqid=0 2024-11-11T18:14:16,943 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/eb972570bddb4429870bf0e86fdf9b42 is 175, key is test_row_0/A:col10/1731348854104/Put/seqid=0 2024-11-11T18:14:16,947 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/C of cc678935c00a295255833d490f0a10a8 into dca2c31cfb5d40f7b9494b4522440c40(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:16,947 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:16,947 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/C, priority=13, startTime=1731348856480; duration=0sec 2024-11-11T18:14:16,947 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:16,947 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:C 2024-11-11T18:14:16,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742091_1267 (size=32175) 2024-11-11T18:14:16,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742092_1268 (size=12301) 2024-11-11T18:14:16,950 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/eafe0b82fce742ad9b64c4b84da793f5 2024-11-11T18:14:16,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/4b6ed538b21a482fa3fc8552d4ba18f6 is 50, key is test_row_0/C:col10/1731348854808/Put/seqid=0 2024-11-11T18:14:16,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742093_1269 (size=12301) 2024-11-11T18:14:16,977 INFO [Thread-751 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-751. 2024-11-11T18:14:16,977 DEBUG [Thread-751 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:16,977 DEBUG [Thread-751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,977 DEBUG [Thread-751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:16,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:17,354 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/eb972570bddb4429870bf0e86fdf9b42 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eb972570bddb4429870bf0e86fdf9b42 2024-11-11T18:14:17,359 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cc678935c00a295255833d490f0a10a8/A of cc678935c00a295255833d490f0a10a8 into eb972570bddb4429870bf0e86fdf9b42(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:17,359 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:17,359 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., storeName=cc678935c00a295255833d490f0a10a8/A, priority=13, startTime=1731348856480; duration=0sec 2024-11-11T18:14:17,359 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:17,359 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cc678935c00a295255833d490f0a10a8:A 2024-11-11T18:14:17,363 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/4b6ed538b21a482fa3fc8552d4ba18f6 2024-11-11T18:14:17,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/fd4e49b7072a48dfa6a0e2f7752af17f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/fd4e49b7072a48dfa6a0e2f7752af17f 2024-11-11T18:14:17,371 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/fd4e49b7072a48dfa6a0e2f7752af17f, entries=150, sequenceid=469, filesize=30.5 K 2024-11-11T18:14:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/eafe0b82fce742ad9b64c4b84da793f5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/eafe0b82fce742ad9b64c4b84da793f5 2024-11-11T18:14:17,374 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/eafe0b82fce742ad9b64c4b84da793f5, entries=150, sequenceid=469, filesize=12.0 K 2024-11-11T18:14:17,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/4b6ed538b21a482fa3fc8552d4ba18f6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/4b6ed538b21a482fa3fc8552d4ba18f6 2024-11-11T18:14:17,378 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/4b6ed538b21a482fa3fc8552d4ba18f6, entries=150, sequenceid=469, filesize=12.0 K 2024-11-11T18:14:17,379 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for cc678935c00a295255833d490f0a10a8 in 884ms, sequenceid=469, compaction requested=false 2024-11-11T18:14:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for cc678935c00a295255833d490f0a10a8: 2024-11-11T18:14:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:17,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T18:14:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T18:14:17,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T18:14:17,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4220 sec 2024-11-11T18:14:17,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.4270 sec 2024-11-11T18:14:18,809 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T18:14:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T18:14:19,114 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:19,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 220 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2230 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2292 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1020 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3059 rows 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1022 2024-11-11T18:14:19,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3065 rows 2024-11-11T18:14:19,115 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:14:19,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:14:19,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:19,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:19,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:19,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T18:14:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T18:14:19,119 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348859119"}]},"ts":"1731348859119"} 2024-11-11T18:14:19,121 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T18:14:19,121 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T18:14:19,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:14:19,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, UNASSIGN}] 2024-11-11T18:14:19,124 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, UNASSIGN 2024-11-11T18:14:19,125 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:19,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, UNASSIGN because future has completed 2024-11-11T18:14:19,126 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:14:19,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:14:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T18:14:19,279 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] handler.UnassignRegionHandler(122): Close cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:19,279 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:14:19,279 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1722): Closing cc678935c00a295255833d490f0a10a8, disabling compactions & flushes 2024-11-11T18:14:19,279 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:19,279 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:19,279 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. after waiting 0 ms 2024-11-11T18:14:19,279 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:19,279 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(2902): Flushing cc678935c00a295255833d490f0a10a8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T18:14:19,280 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=A 2024-11-11T18:14:19,280 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:19,280 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=B 2024-11-11T18:14:19,280 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:19,280 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cc678935c00a295255833d490f0a10a8, store=C 2024-11-11T18:14:19,280 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:19,288 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ef9a89efeb424acfb25f11660710252f_cc678935c00a295255833d490f0a10a8 is 50, key is test_row_0/A:col10/1731348856975/Put/seqid=0 2024-11-11T18:14:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742094_1270 (size=12454) 2024-11-11T18:14:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T18:14:19,692 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:19,696 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ef9a89efeb424acfb25f11660710252f_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ef9a89efeb424acfb25f11660710252f_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:19,697 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bd25e6ca829f43888da87f6ea2f957b5, store: [table=TestAcidGuarantees family=A region=cc678935c00a295255833d490f0a10a8] 2024-11-11T18:14:19,698 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bd25e6ca829f43888da87f6ea2f957b5 is 175, key is test_row_0/A:col10/1731348856975/Put/seqid=0 2024-11-11T18:14:19,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742095_1271 (size=31255) 2024-11-11T18:14:19,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T18:14:20,106 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=480, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bd25e6ca829f43888da87f6ea2f957b5 2024-11-11T18:14:20,113 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/5f068c1f731c4c988faf0973fd56a3e7 is 50, key is test_row_0/B:col10/1731348856975/Put/seqid=0 2024-11-11T18:14:20,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742096_1272 (size=12301) 2024-11-11T18:14:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T18:14:20,518 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/5f068c1f731c4c988faf0973fd56a3e7 2024-11-11T18:14:20,526 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/2fb736cbbc10405a861113620ac41d7b is 50, key is test_row_0/C:col10/1731348856975/Put/seqid=0 2024-11-11T18:14:20,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742097_1273 (size=12301) 2024-11-11T18:14:20,935 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/2fb736cbbc10405a861113620ac41d7b 2024-11-11T18:14:20,940 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/A/bd25e6ca829f43888da87f6ea2f957b5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bd25e6ca829f43888da87f6ea2f957b5 2024-11-11T18:14:20,945 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bd25e6ca829f43888da87f6ea2f957b5, entries=150, sequenceid=480, filesize=30.5 K 2024-11-11T18:14:20,946 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/B/5f068c1f731c4c988faf0973fd56a3e7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5f068c1f731c4c988faf0973fd56a3e7 2024-11-11T18:14:20,951 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5f068c1f731c4c988faf0973fd56a3e7, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T18:14:20,951 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/.tmp/C/2fb736cbbc10405a861113620ac41d7b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2fb736cbbc10405a861113620ac41d7b 2024-11-11T18:14:20,959 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2fb736cbbc10405a861113620ac41d7b, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T18:14:20,960 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cc678935c00a295255833d490f0a10a8 in 1681ms, sequenceid=480, compaction requested=true 2024-11-11T18:14:20,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/4521633d6e9e4fe9a318cb4a3862234b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91ca3b2bb8e147e98cfbe9346f02da80, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8b8d577c0b5849df98373dd1ea162dcd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/58d988f6ad5b41f49295b555ee2c8788, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/923eedfce4b648e789dc793146d2db64, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/528c3dadb88647dfb708054019e886bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ccefdefb520c494f83ae6983abe947e5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8] to archive 2024-11-11T18:14:20,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:14:20,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5437754b4f404cba8cf40dc1abc970ec 2024-11-11T18:14:20,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0a4e678e0d6946bcacc76160748626f5 2024-11-11T18:14:20,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/4521633d6e9e4fe9a318cb4a3862234b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/4521633d6e9e4fe9a318cb4a3862234b 2024-11-11T18:14:20,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/95f8cbba22044e9c96b6d7c9f3d1f3e0 2024-11-11T18:14:20,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/0c3c839a1909406e9bf67c1d1f8aa132 2024-11-11T18:14:20,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/88c2abcb8db04ff5bad30282ae760209 2024-11-11T18:14:20,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/251cb6dc7f7841b985aca9cc169b2925 2024-11-11T18:14:20,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91ca3b2bb8e147e98cfbe9346f02da80 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91ca3b2bb8e147e98cfbe9346f02da80 2024-11-11T18:14:20,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ebf3b14ee00c4d57ad2ee0f6e8b7a0ff 2024-11-11T18:14:20,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/218a6914cf284f3a8dba9d6dec510bc9 2024-11-11T18:14:20,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/91b62bb75f574b32b511ddb5c6030ffe 2024-11-11T18:14:20,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8b8d577c0b5849df98373dd1ea162dcd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8b8d577c0b5849df98373dd1ea162dcd 2024-11-11T18:14:20,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/5c7af1c27173434c8493cac5bd8458a5 2024-11-11T18:14:20,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/58d988f6ad5b41f49295b555ee2c8788 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/58d988f6ad5b41f49295b555ee2c8788 2024-11-11T18:14:20,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eec7c264ec37416695aeeb6d2f9f4d43 2024-11-11T18:14:20,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e6e7e00e439e4c519b5817ba38139da4 2024-11-11T18:14:20,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/e788f301c9b1455289054e13e98d3f46 2024-11-11T18:14:20,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/923eedfce4b648e789dc793146d2db64 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/923eedfce4b648e789dc793146d2db64 2024-11-11T18:14:20,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bc25198deedd47f3bdefe9fb28e4b631 2024-11-11T18:14:20,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/46f6a34af32342cc9dc8e9272588fe6e 2024-11-11T18:14:20,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/61ba5bf092fd4487bf7812afdaf00900 2024-11-11T18:14:20,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/528c3dadb88647dfb708054019e886bb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/528c3dadb88647dfb708054019e886bb 2024-11-11T18:14:21,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/d126ee1f03e043269540a0ca857f57f2 2024-11-11T18:14:21,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/8d5028253a5748b581dec9d9429bffb8 2024-11-11T18:14:21,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ccefdefb520c494f83ae6983abe947e5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/ccefdefb520c494f83ae6983abe947e5 2024-11-11T18:14:21,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/b193a10fe02c4c8a9a9d05b5fc70a784 2024-11-11T18:14:21,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/817eec6b3f8a476585bed82404fab2e8 2024-11-11T18:14:21,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/6f7430d12ef94ef8b07688047f164852, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ee9ebce9116f46169ca0da8865527848, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f386b144f6c944d0b30a97af2351425b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4fb792b1381e4c3d9c8bfbb7749188e9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c0e1ceeb1ac1460db1a988bdd24dca1f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/d24d2585af4b4f55ae7bbeb8454897c1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/422483e769b043f09dcfaa9d908e0508, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f691c60faa5c433a9fd77a1ca772ea6f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/53952ba289194bfe9daf40fd25a1b2ed, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dfd2be811a924a3eba272ab5eb41d91c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/36b3c5ee684b43759cd628f5a055c5c3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/2b143124fbcf423091725b748dd76824, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5cc99770e9c54e9d9ea6ec2931c32f22, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ea2d127699f848c3b9fb8a8ee19f1540, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dd2c2fbcb8d64829b026d587d70e2bd7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/911e26869d8a407bae3dd44ada9a861a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f6ce4d710d4e4862be62d12104c01e96, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c4f4c26677a54084948851bc6cb9960b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/92d6c1a9daf84808a5c9342a57750f95, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/42a9a667dfbe4b609ecee07700366979, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/aab9985b610a4eac8a4f8ecf73ab0c00, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/3c36de2f28eb4b748b477da00294d441, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4d15f044e6314ff7a6a2c5bbea51cdc9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f861caab6ffd46138d54a6ab26e0e706, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/623e746f97294f93b3a31d907da9cbd6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/43d0c6f088ec44339be37a2815c3f5e9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/0ae793c369a744a68fde7d2548e47d57] to archive 2024-11-11T18:14:21,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:14:21,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/6f7430d12ef94ef8b07688047f164852 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/6f7430d12ef94ef8b07688047f164852 2024-11-11T18:14:21,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ee9ebce9116f46169ca0da8865527848 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ee9ebce9116f46169ca0da8865527848 2024-11-11T18:14:21,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f386b144f6c944d0b30a97af2351425b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f386b144f6c944d0b30a97af2351425b 2024-11-11T18:14:21,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4fb792b1381e4c3d9c8bfbb7749188e9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4fb792b1381e4c3d9c8bfbb7749188e9 2024-11-11T18:14:21,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c0e1ceeb1ac1460db1a988bdd24dca1f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c0e1ceeb1ac1460db1a988bdd24dca1f 2024-11-11T18:14:21,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/d24d2585af4b4f55ae7bbeb8454897c1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/d24d2585af4b4f55ae7bbeb8454897c1 2024-11-11T18:14:21,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/422483e769b043f09dcfaa9d908e0508 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/422483e769b043f09dcfaa9d908e0508 2024-11-11T18:14:21,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f691c60faa5c433a9fd77a1ca772ea6f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f691c60faa5c433a9fd77a1ca772ea6f 2024-11-11T18:14:21,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/53952ba289194bfe9daf40fd25a1b2ed to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/53952ba289194bfe9daf40fd25a1b2ed 2024-11-11T18:14:21,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dfd2be811a924a3eba272ab5eb41d91c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dfd2be811a924a3eba272ab5eb41d91c 2024-11-11T18:14:21,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/36b3c5ee684b43759cd628f5a055c5c3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/36b3c5ee684b43759cd628f5a055c5c3 2024-11-11T18:14:21,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/2b143124fbcf423091725b748dd76824 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/2b143124fbcf423091725b748dd76824 2024-11-11T18:14:21,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5cc99770e9c54e9d9ea6ec2931c32f22 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5cc99770e9c54e9d9ea6ec2931c32f22 2024-11-11T18:14:21,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ea2d127699f848c3b9fb8a8ee19f1540 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/ea2d127699f848c3b9fb8a8ee19f1540 2024-11-11T18:14:21,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dd2c2fbcb8d64829b026d587d70e2bd7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/dd2c2fbcb8d64829b026d587d70e2bd7 2024-11-11T18:14:21,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/911e26869d8a407bae3dd44ada9a861a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/911e26869d8a407bae3dd44ada9a861a 2024-11-11T18:14:21,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f6ce4d710d4e4862be62d12104c01e96 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f6ce4d710d4e4862be62d12104c01e96 2024-11-11T18:14:21,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c4f4c26677a54084948851bc6cb9960b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/c4f4c26677a54084948851bc6cb9960b 2024-11-11T18:14:21,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/92d6c1a9daf84808a5c9342a57750f95 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/92d6c1a9daf84808a5c9342a57750f95 2024-11-11T18:14:21,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/42a9a667dfbe4b609ecee07700366979 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/42a9a667dfbe4b609ecee07700366979 2024-11-11T18:14:21,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/aab9985b610a4eac8a4f8ecf73ab0c00 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/aab9985b610a4eac8a4f8ecf73ab0c00 2024-11-11T18:14:21,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/3c36de2f28eb4b748b477da00294d441 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/3c36de2f28eb4b748b477da00294d441 2024-11-11T18:14:21,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4d15f044e6314ff7a6a2c5bbea51cdc9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/4d15f044e6314ff7a6a2c5bbea51cdc9 2024-11-11T18:14:21,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f861caab6ffd46138d54a6ab26e0e706 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/f861caab6ffd46138d54a6ab26e0e706 2024-11-11T18:14:21,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/623e746f97294f93b3a31d907da9cbd6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/623e746f97294f93b3a31d907da9cbd6 2024-11-11T18:14:21,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/43d0c6f088ec44339be37a2815c3f5e9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/43d0c6f088ec44339be37a2815c3f5e9 2024-11-11T18:14:21,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/0ae793c369a744a68fde7d2548e47d57 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/0ae793c369a744a68fde7d2548e47d57 2024-11-11T18:14:21,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/858b78358fd34e97a7af132b80a60c1e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ca2faecf23804e17bcef38ae28470065, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/f510061cdd1d4bb7beb753cad85d5e9b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/49f454527a80467ea267c238948b47c8, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/14ccfab7744e49a4890c54ffb979bf75, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/10a3dc191ead46239079ef798c77cfa3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/32bbe863f1ee4a93b1b38019650764a2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8dc9aeea8de94c3595b3636fa60af3c1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b075e4f871e84dd6be118c3fdec5c8c7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8039e2b059964a67bbe045cd6290fe1a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/3b9190fe6b06485e81b59d2bad69bc77, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7bc2acc30ce64e01b5411812d4c1b33f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5bc69a32e8f54ba8be4999d845e8f7ed, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7275c51ccd424b03a22eab42bd3a8d71, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b028b93da5a2409d8975067825dca1c0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5fdb7f2d5d05498f8ba54a2950cb92b7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ff5db462cfdb4dfb8fec2df31ca1f756, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/64e8b121aefa487f834dccc9f81c255f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/97be8961a67c4044aebfb61a6b7b3b82, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/50c7c9ab66674ecf921c07d2c585032f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/baa85b0588eb45aba5bfbbf8ee9b70e6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2f358f2f121c4d93b05347d3cd063535, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/9d07015b754b41ac860cd5e24fb310ad, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/493c54bb121b47769497373300798ab2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8ceb5e88ebe34a51a5310e6a0b39ab98, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7df61ebb958944ce9d414629e4e0e2bf] to archive 2024-11-11T18:14:21,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:14:21,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/858b78358fd34e97a7af132b80a60c1e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/858b78358fd34e97a7af132b80a60c1e 2024-11-11T18:14:21,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ca2faecf23804e17bcef38ae28470065 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ca2faecf23804e17bcef38ae28470065 2024-11-11T18:14:21,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/f510061cdd1d4bb7beb753cad85d5e9b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/f510061cdd1d4bb7beb753cad85d5e9b 2024-11-11T18:14:21,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/49f454527a80467ea267c238948b47c8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/49f454527a80467ea267c238948b47c8 2024-11-11T18:14:21,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/14ccfab7744e49a4890c54ffb979bf75 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/14ccfab7744e49a4890c54ffb979bf75 2024-11-11T18:14:21,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/10a3dc191ead46239079ef798c77cfa3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/10a3dc191ead46239079ef798c77cfa3 2024-11-11T18:14:21,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/32bbe863f1ee4a93b1b38019650764a2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/32bbe863f1ee4a93b1b38019650764a2 2024-11-11T18:14:21,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8dc9aeea8de94c3595b3636fa60af3c1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8dc9aeea8de94c3595b3636fa60af3c1 2024-11-11T18:14:21,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b075e4f871e84dd6be118c3fdec5c8c7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b075e4f871e84dd6be118c3fdec5c8c7 2024-11-11T18:14:21,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8039e2b059964a67bbe045cd6290fe1a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8039e2b059964a67bbe045cd6290fe1a 2024-11-11T18:14:21,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/3b9190fe6b06485e81b59d2bad69bc77 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/3b9190fe6b06485e81b59d2bad69bc77 2024-11-11T18:14:21,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7bc2acc30ce64e01b5411812d4c1b33f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7bc2acc30ce64e01b5411812d4c1b33f 2024-11-11T18:14:21,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5bc69a32e8f54ba8be4999d845e8f7ed to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5bc69a32e8f54ba8be4999d845e8f7ed 2024-11-11T18:14:21,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7275c51ccd424b03a22eab42bd3a8d71 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7275c51ccd424b03a22eab42bd3a8d71 2024-11-11T18:14:21,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b028b93da5a2409d8975067825dca1c0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/b028b93da5a2409d8975067825dca1c0 2024-11-11T18:14:21,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5fdb7f2d5d05498f8ba54a2950cb92b7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/5fdb7f2d5d05498f8ba54a2950cb92b7 2024-11-11T18:14:21,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ff5db462cfdb4dfb8fec2df31ca1f756 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/ff5db462cfdb4dfb8fec2df31ca1f756 2024-11-11T18:14:21,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/64e8b121aefa487f834dccc9f81c255f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/64e8b121aefa487f834dccc9f81c255f 2024-11-11T18:14:21,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/97be8961a67c4044aebfb61a6b7b3b82 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/97be8961a67c4044aebfb61a6b7b3b82 2024-11-11T18:14:21,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/50c7c9ab66674ecf921c07d2c585032f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/50c7c9ab66674ecf921c07d2c585032f 2024-11-11T18:14:21,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/baa85b0588eb45aba5bfbbf8ee9b70e6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/baa85b0588eb45aba5bfbbf8ee9b70e6 2024-11-11T18:14:21,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2f358f2f121c4d93b05347d3cd063535 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2f358f2f121c4d93b05347d3cd063535 2024-11-11T18:14:21,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/9d07015b754b41ac860cd5e24fb310ad to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/9d07015b754b41ac860cd5e24fb310ad 2024-11-11T18:14:21,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/493c54bb121b47769497373300798ab2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/493c54bb121b47769497373300798ab2 2024-11-11T18:14:21,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8ceb5e88ebe34a51a5310e6a0b39ab98 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/8ceb5e88ebe34a51a5310e6a0b39ab98 2024-11-11T18:14:21,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7df61ebb958944ce9d414629e4e0e2bf to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/7df61ebb958944ce9d414629e4e0e2bf 2024-11-11T18:14:21,088 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/recovered.edits/483.seqid, newMaxSeqId=483, maxSeqId=4 2024-11-11T18:14:21,089 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. 2024-11-11T18:14:21,089 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] regionserver.HRegion(1676): Region close journal for cc678935c00a295255833d490f0a10a8: Waiting for close lock at 1731348859279Running coprocessor pre-close hooks at 1731348859279Disabling compacts and flushes for region at 1731348859279Disabling writes for close at 1731348859279Obtaining lock to block concurrent updates at 1731348859279Preparing flush snapshotting stores in cc678935c00a295255833d490f0a10a8 at 1731348859279Finished memstore snapshotting TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731348859280 (+1 ms)Flushing stores of TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8. at 1731348859281 (+1 ms)Flushing cc678935c00a295255833d490f0a10a8/A: creating writer at 1731348859281Flushing mob file cc678935c00a295255833d490f0a10a8/A: appending metadata at 1731348859287 (+6 ms)Flushing mob file cc678935c00a295255833d490f0a10a8/A: closing flushed file at 1731348859287Flushing cc678935c00a295255833d490f0a10a8/A: appending metadata at 1731348859697 (+410 ms)Flushing cc678935c00a295255833d490f0a10a8/A: closing flushed file at 1731348859698 (+1 ms)Flushing cc678935c00a295255833d490f0a10a8/B: creating writer at 1731348860110 (+412 ms)Flushing cc678935c00a295255833d490f0a10a8/B: appending metadata at 1731348860113 (+3 ms)Flushing cc678935c00a295255833d490f0a10a8/B: closing flushed file at 1731348860113Flushing cc678935c00a295255833d490f0a10a8/C: creating writer at 1731348860522 (+409 ms)Flushing cc678935c00a295255833d490f0a10a8/C: appending metadata at 1731348860525 (+3 ms)Flushing cc678935c00a295255833d490f0a10a8/C: closing flushed file at 1731348860525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fa69e35: reopening flushed file at 1731348860939 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68c40e45: reopening flushed file at 1731348860945 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c38d180: reopening flushed file at 1731348860951 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cc678935c00a295255833d490f0a10a8 in 1681ms, sequenceid=480, compaction requested=true at 1731348860960 (+9 ms)Writing region close event to WAL at 1731348861077 (+117 ms)Running coprocessor post-close hooks at 1731348861089 (+12 ms)Closed at 1731348861089 2024-11-11T18:14:21,092 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=69}] handler.UnassignRegionHandler(157): Closed cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=cc678935c00a295255833d490f0a10a8, regionState=CLOSED 2024-11-11T18:14:21,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; CloseRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:14:21,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T18:14:21,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; CloseRegionProcedure cc678935c00a295255833d490f0a10a8, server=90f34c5bee6a,42957,1731348800579 in 1.9690 sec 2024-11-11T18:14:21,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-11T18:14:21,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cc678935c00a295255833d490f0a10a8, UNASSIGN in 1.9740 sec 2024-11-11T18:14:21,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T18:14:21,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9800 sec 2024-11-11T18:14:21,106 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348861105"}]},"ts":"1731348861105"} 2024-11-11T18:14:21,107 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T18:14:21,107 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T18:14:21,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9920 sec 2024-11-11T18:14:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T18:14:21,265 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:21,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T18:14:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,268 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=70, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,268 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=70, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T18:14:21,271 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,274 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/recovered.edits] 2024-11-11T18:14:21,277 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bd25e6ca829f43888da87f6ea2f957b5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/bd25e6ca829f43888da87f6ea2f957b5 2024-11-11T18:14:21,278 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eb972570bddb4429870bf0e86fdf9b42 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/eb972570bddb4429870bf0e86fdf9b42 2024-11-11T18:14:21,280 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/fd4e49b7072a48dfa6a0e2f7752af17f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/A/fd4e49b7072a48dfa6a0e2f7752af17f 2024-11-11T18:14:21,282 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5f068c1f731c4c988faf0973fd56a3e7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/5f068c1f731c4c988faf0973fd56a3e7 2024-11-11T18:14:21,284 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/990014b389f542e79b663323319fc75f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/990014b389f542e79b663323319fc75f 2024-11-11T18:14:21,285 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/eafe0b82fce742ad9b64c4b84da793f5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/B/eafe0b82fce742ad9b64c4b84da793f5 2024-11-11T18:14:21,287 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2fb736cbbc10405a861113620ac41d7b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/2fb736cbbc10405a861113620ac41d7b 2024-11-11T18:14:21,288 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/4b6ed538b21a482fa3fc8552d4ba18f6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/4b6ed538b21a482fa3fc8552d4ba18f6 2024-11-11T18:14:21,290 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/dca2c31cfb5d40f7b9494b4522440c40 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/C/dca2c31cfb5d40f7b9494b4522440c40 2024-11-11T18:14:21,292 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/recovered.edits/483.seqid to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8/recovered.edits/483.seqid 2024-11-11T18:14:21,293 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,293 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T18:14:21,293 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T18:14:21,294 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T18:14:21,298 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a3df66840c6495da1d822cfc7fbccb0_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111a3df66840c6495da1d822cfc7fbccb0_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,300 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111231f95a546994945bd85a0b29d6f2364_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111231f95a546994945bd85a0b29d6f2364_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,301 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111123d9745d18ed45969d7db61cdeb60da0_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111123d9745d18ed45969d7db61cdeb60da0_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,302 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124f3dff788de4d4980b8a252f8806e44_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124f3dff788de4d4980b8a252f8806e44_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,303 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111128e8b2ceec3541179231bcaa21f5d962_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111128e8b2ceec3541179231bcaa21f5d962_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,304 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111299a7dbb33f94948997b82dbcbdc10d6_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111299a7dbb33f94948997b82dbcbdc10d6_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,306 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111142b8ec299c644c9092b2847ca6e454a4_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111142b8ec299c644c9092b2847ca6e454a4_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,307 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114445a8ac3b3a465895abdc41f39bf8ef_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114445a8ac3b3a465895abdc41f39bf8ef_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,308 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166d727d2a41b48f19db992172e47d899_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111166d727d2a41b48f19db992172e47d899_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,310 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111719234609e4d403396f1f0b7a3118baa_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111719234609e4d403396f1f0b7a3118baa_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,311 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111793a5ca3ca71409ca865c3cce7a210bb_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111793a5ca3ca71409ca865c3cce7a210bb_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,312 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111181440c46844b494d9fd2f2c3b67d518f_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111181440c46844b494d9fd2f2c3b67d518f_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,313 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b3ead4ec08b4fb9a3bb2eafb0e4bcd1_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b3ead4ec08b4fb9a3bb2eafb0e4bcd1_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,314 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b4d6d9d3aad74347b73eeb57149c49fb_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b4d6d9d3aad74347b73eeb57149c49fb_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,315 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bc8a0f07fbf242b381b2ffc1eb21f3a0_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bc8a0f07fbf242b381b2ffc1eb21f3a0_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,316 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bd7dd44c3ac8410dbc36aa82955e1be4_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bd7dd44c3ac8410dbc36aa82955e1be4_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,317 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c91e666de8fc49feaa097da81fc9a987_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c91e666de8fc49feaa097da81fc9a987_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,319 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e05e6118236b44ebab6ccd02b0b8be5f_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e05e6118236b44ebab6ccd02b0b8be5f_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,320 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e073fc9b58dc4a818d12ba7d775689f8_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e073fc9b58dc4a818d12ba7d775689f8_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,322 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ef9a89efeb424acfb25f11660710252f_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ef9a89efeb424acfb25f11660710252f_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,323 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4fda6849aca49718f9d8199f061cbf2_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f4fda6849aca49718f9d8199f061cbf2_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f914931a7e4d45de922404d87b8aa60c_cc678935c00a295255833d490f0a10a8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f914931a7e4d45de922404d87b8aa60c_cc678935c00a295255833d490f0a10a8 2024-11-11T18:14:21,325 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T18:14:21,327 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=70, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,329 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T18:14:21,332 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T18:14:21,333 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=70, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,333 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T18:14:21,333 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731348861333"}]},"ts":"9223372036854775807"} 2024-11-11T18:14:21,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T18:14:21,336 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cc678935c00a295255833d490f0a10a8, NAME => 'TestAcidGuarantees,,1731348832409.cc678935c00a295255833d490f0a10a8.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T18:14:21,336 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T18:14:21,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731348861336"}]},"ts":"9223372036854775807"} 2024-11-11T18:14:21,339 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T18:14:21,339 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=70, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 74 msec 2024-11-11T18:14:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T18:14:21,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T18:14:21,375 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:21,385 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 249) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1702191631_22 at /127.0.0.1:45958 [Waiting for operation #841] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1088901640_22 at /127.0.0.1:35064 [Waiting for operation #267] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1088901640_22 at /127.0.0.1:58346 [Waiting for operation #466] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1702191631_22 at /127.0.0.1:45982 [Waiting for operation #529] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=538 (was 503) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8521 (was 8792) 2024-11-11T18:14:21,393 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=538, ProcessCount=11, AvailableMemoryMB=8520 2024-11-11T18:14:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:14:21,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:14:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:21,397 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T18:14:21,397 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:21,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 71 2024-11-11T18:14:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-11T18:14:21,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T18:14:21,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742098_1274 (size=1061) 2024-11-11T18:14:21,405 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:14:21,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742099_1275 (size=53) 2024-11-11T18:14:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-11T18:14:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-11T18:14:21,810 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:14:21,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7ac34b3c9a264f2c66d52e98e093bf68, disabling compactions & flushes 2024-11-11T18:14:21,811 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:21,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:21,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. after waiting 0 ms 2024-11-11T18:14:21,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:21,811 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:21,811 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ac34b3c9a264f2c66d52e98e093bf68: Waiting for close lock at 1731348861811Disabling compacts and flushes for region at 1731348861811Disabling writes for close at 1731348861811Writing region close event to WAL at 1731348861811Closed at 1731348861811 2024-11-11T18:14:21,812 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T18:14:21,813 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731348861812"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731348861812"}]},"ts":"1731348861812"} 2024-11-11T18:14:21,814 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T18:14:21,815 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T18:14:21,815 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348861815"}]},"ts":"1731348861815"} 2024-11-11T18:14:21,817 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T18:14:21,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, ASSIGN}] 2024-11-11T18:14:21,818 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, ASSIGN 2024-11-11T18:14:21,819 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, ASSIGN; state=OFFLINE, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=false 2024-11-11T18:14:21,970 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=7ac34b3c9a264f2c66d52e98e093bf68, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:21,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, ASSIGN because future has completed 2024-11-11T18:14:21,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:14:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-11T18:14:22,126 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,126 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:14:22,127 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,127 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:14:22,127 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,127 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,128 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,129 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:22,129 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ac34b3c9a264f2c66d52e98e093bf68 columnFamilyName A 2024-11-11T18:14:22,129 DEBUG [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:22,130 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.HStore(327): Store=7ac34b3c9a264f2c66d52e98e093bf68/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:22,130 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,131 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:22,131 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ac34b3c9a264f2c66d52e98e093bf68 columnFamilyName B 2024-11-11T18:14:22,131 DEBUG [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:22,131 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.HStore(327): Store=7ac34b3c9a264f2c66d52e98e093bf68/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:22,131 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,132 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:22,132 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ac34b3c9a264f2c66d52e98e093bf68 columnFamilyName C 2024-11-11T18:14:22,132 DEBUG [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:22,133 INFO [StoreOpener-7ac34b3c9a264f2c66d52e98e093bf68-1 {}] regionserver.HStore(327): Store=7ac34b3c9a264f2c66d52e98e093bf68/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:22,133 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,133 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,133 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,133 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,134 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,134 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,135 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:14:22,136 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,137 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:14:22,138 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 7ac34b3c9a264f2c66d52e98e093bf68; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59719252, jitterRate=-0.11011379957199097}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:14:22,138 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,138 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 7ac34b3c9a264f2c66d52e98e093bf68: Running coprocessor pre-open hook at 1731348862127Writing region info on filesystem at 1731348862127Initializing all the Stores at 1731348862128 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348862128Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348862128Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348862128Cleaning up temporary data from old regions at 1731348862134 (+6 ms)Running coprocessor post-open hooks at 1731348862138 (+4 ms)Region opened successfully at 1731348862138 2024-11-11T18:14:22,139 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., pid=73, masterSystemTime=1731348862124 2024-11-11T18:14:22,140 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,140 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,141 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=7ac34b3c9a264f2c66d52e98e093bf68, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:14:22,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T18:14:22,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 in 171 msec 2024-11-11T18:14:22,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-11-11T18:14:22,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, ASSIGN in 327 msec 2024-11-11T18:14:22,146 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T18:14:22,147 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348862146"}]},"ts":"1731348862146"} 2024-11-11T18:14:22,148 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T18:14:22,148 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=71, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T18:14:22,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 753 msec 2024-11-11T18:14:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-11-11T18:14:22,535 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:22,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba7ffab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,537 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26727c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,539 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ca6fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,539 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,542 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:14:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:14:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e226854 2024-11-11T18:14:22,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:14:22,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:14:22,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11961dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424a3bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36548c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48345386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d3ed5ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,551 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb92ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ddee8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358cbd5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,554 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a693b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5138abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bfe7c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f505fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b7e7d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e15e01c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10b57ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f59873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289dbcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,564 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f6fb010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,565 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661d070b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,569 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e44681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357b0846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478bceb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10961e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3df568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63eadf7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:22,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:22,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:22,574 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f41e5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:22,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:22,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d684916, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:22,576 DEBUG [Thread-1264 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,576 DEBUG [Thread-1262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T18:14:22,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:22,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T18:14:22,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:22,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:22,580 DEBUG [Thread-1273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,585 DEBUG [Thread-1275 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,588 DEBUG [Thread-1258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,589 DEBUG [Thread-1260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:22,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,598 DEBUG [Thread-1266 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348862607, max timestamp=1731348862611Num uniques 859; 2024-11-11T18:14:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348862607, max timestamp=1731348862611Num uniques 859; 2024-11-11T18:14:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348862607, max timestamp=1731348862611Num uniques 859; 2024-11-11T18:14:22,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:22,617 DEBUG [Thread-1269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,618 DEBUG [Thread-1277 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,618 DEBUG [Thread-1271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:22,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:22,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:14:22,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/c0e9e9c731594a9b894ac5b4c57ab737 is 50, key is test_row_0/A:col10/1731348862584/Put/seqid=0 2024-11-11T18:14:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348922642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348922644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348922647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348922647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348922649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742100_1276 (size=12001) 2024-11-11T18:14:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T18:14:22,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T18:14:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348922755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348922755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348922764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348922764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348922764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T18:14:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:22,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:22,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:22,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T18:14:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348922964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348922964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348922975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348922976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:22,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348922976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:22,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T18:14:23,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:23,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:23,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:23,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:23,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:23,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/c0e9e9c731594a9b894ac5b4c57ab737 2024-11-11T18:14:23,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5f2dc8e1d2a24f3aaf647d84b49830d9 is 50, key is test_row_0/B:col10/1731348862584/Put/seqid=0 2024-11-11T18:14:23,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742101_1277 (size=12001) 2024-11-11T18:14:23,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5f2dc8e1d2a24f3aaf647d84b49830d9 2024-11-11T18:14:23,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/29dda5da037c4375993a5256a342bd5f is 50, key is test_row_0/C:col10/1731348862584/Put/seqid=0 2024-11-11T18:14:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742102_1278 (size=12001) 2024-11-11T18:14:23,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/29dda5da037c4375993a5256a342bd5f 2024-11-11T18:14:23,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/c0e9e9c731594a9b894ac5b4c57ab737 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/c0e9e9c731594a9b894ac5b4c57ab737 2024-11-11T18:14:23,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/c0e9e9c731594a9b894ac5b4c57ab737, entries=150, sequenceid=11, filesize=11.7 K 2024-11-11T18:14:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5f2dc8e1d2a24f3aaf647d84b49830d9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5f2dc8e1d2a24f3aaf647d84b49830d9 2024-11-11T18:14:23,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5f2dc8e1d2a24f3aaf647d84b49830d9, entries=150, sequenceid=11, filesize=11.7 K 2024-11-11T18:14:23,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/29dda5da037c4375993a5256a342bd5f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/29dda5da037c4375993a5256a342bd5f 2024-11-11T18:14:23,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/29dda5da037c4375993a5256a342bd5f, entries=150, sequenceid=11, filesize=11.7 K 2024-11-11T18:14:23,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 7ac34b3c9a264f2c66d52e98e093bf68 in 598ms, sequenceid=11, compaction requested=false 2024-11-11T18:14:23,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:23,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:23,197 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:23,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T18:14:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/9edd1fb214644830a50de78077726a82 is 50, key is test_row_0/A:col10/1731348862644/Put/seqid=0 2024-11-11T18:14:23,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742103_1279 (size=12001) 2024-11-11T18:14:23,252 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/9edd1fb214644830a50de78077726a82 2024-11-11T18:14:23,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/85965d824b5440d197926e45fa7f3e9a is 50, key is test_row_0/B:col10/1731348862644/Put/seqid=0 2024-11-11T18:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:23,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:23,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348923289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348923291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348923292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348923293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348923293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742104_1280 (size=12001) 2024-11-11T18:14:23,315 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/85965d824b5440d197926e45fa7f3e9a 2024-11-11T18:14:23,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/1da070f975664cef9954c32c54bf95de is 50, key is test_row_0/C:col10/1731348862644/Put/seqid=0 2024-11-11T18:14:23,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742105_1281 (size=12001) 2024-11-11T18:14:23,362 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/1da070f975664cef9954c32c54bf95de 2024-11-11T18:14:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/9edd1fb214644830a50de78077726a82 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/9edd1fb214644830a50de78077726a82 2024-11-11T18:14:23,375 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/9edd1fb214644830a50de78077726a82, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T18:14:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/85965d824b5440d197926e45fa7f3e9a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/85965d824b5440d197926e45fa7f3e9a 2024-11-11T18:14:23,401 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/85965d824b5440d197926e45fa7f3e9a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T18:14:23,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/1da070f975664cef9954c32c54bf95de as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/1da070f975664cef9954c32c54bf95de 2024-11-11T18:14:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348923405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,412 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/1da070f975664cef9954c32c54bf95de, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T18:14:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,414 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 7ac34b3c9a264f2c66d52e98e093bf68 in 217ms, sequenceid=48, compaction requested=false 2024-11-11T18:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:23,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T18:14:23,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T18:14:23,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T18:14:23,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 838 msec 2024-11-11T18:14:23,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 843 msec 2024-11-11T18:14:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:23,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:23,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:23,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/1ff6e61edc3149248f0e93bcb54c8857 is 50, key is test_row_0/A:col10/1731348863615/Put/seqid=0 2024-11-11T18:14:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742106_1282 (size=14341) 2024-11-11T18:14:23,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/1ff6e61edc3149248f0e93bcb54c8857 2024-11-11T18:14:23,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5019139788e34a4ba98772e05a6a49ac is 50, key is test_row_0/B:col10/1731348863615/Put/seqid=0 2024-11-11T18:14:23,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348923645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348863629, max timestamp=1731348863648Num uniques 859; 2024-11-11T18:14:23,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348863629, max timestamp=1731348863648Num uniques 859; 2024-11-11T18:14:23,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:23,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:23,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348863629, max timestamp=1731348863648Num uniques 859; 2024-11-11T18:14:23,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742107_1283 (size=12001) 2024-11-11T18:14:23,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348923685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348923684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348923687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348923688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T18:14:23,715 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:23,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T18:14:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T18:14:23,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:23,718 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:23,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348923755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348923805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348923805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348923805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348923805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T18:14:23,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T18:14:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:23,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:23,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348923965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:23,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348924015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348924015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348924015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348924016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T18:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:24,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T18:14:24,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5019139788e34a4ba98772e05a6a49ac 2024-11-11T18:14:24,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/189abcc1ae124ca5ad105f8aa615a62c is 50, key is test_row_0/C:col10/1731348863615/Put/seqid=0 2024-11-11T18:14:24,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742108_1284 (size=12001) 2024-11-11T18:14:24,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/189abcc1ae124ca5ad105f8aa615a62c 2024-11-11T18:14:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/1ff6e61edc3149248f0e93bcb54c8857 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1ff6e61edc3149248f0e93bcb54c8857 2024-11-11T18:14:24,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1ff6e61edc3149248f0e93bcb54c8857, entries=200, sequenceid=59, filesize=14.0 K 2024-11-11T18:14:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5019139788e34a4ba98772e05a6a49ac as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5019139788e34a4ba98772e05a6a49ac 2024-11-11T18:14:24,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5019139788e34a4ba98772e05a6a49ac, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T18:14:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/189abcc1ae124ca5ad105f8aa615a62c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/189abcc1ae124ca5ad105f8aa615a62c 2024-11-11T18:14:24,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/189abcc1ae124ca5ad105f8aa615a62c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T18:14:24,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7ac34b3c9a264f2c66d52e98e093bf68 in 483ms, sequenceid=59, compaction requested=true 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:24,101 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:24,101 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:24,102 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:24,102 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:24,102 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:24,102 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:24,102 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,102 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,102 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5f2dc8e1d2a24f3aaf647d84b49830d9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/85965d824b5440d197926e45fa7f3e9a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5019139788e34a4ba98772e05a6a49ac] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=35.2 K 2024-11-11T18:14:24,102 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/c0e9e9c731594a9b894ac5b4c57ab737, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/9edd1fb214644830a50de78077726a82, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1ff6e61edc3149248f0e93bcb54c8857] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=37.4 K 2024-11-11T18:14:24,103 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f2dc8e1d2a24f3aaf647d84b49830d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731348862584 2024-11-11T18:14:24,105 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 85965d824b5440d197926e45fa7f3e9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731348862640 2024-11-11T18:14:24,105 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e9e9c731594a9b894ac5b4c57ab737, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731348862584 2024-11-11T18:14:24,105 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5019139788e34a4ba98772e05a6a49ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348863290 2024-11-11T18:14:24,105 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9edd1fb214644830a50de78077726a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731348862640 2024-11-11T18:14:24,107 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff6e61edc3149248f0e93bcb54c8857, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348863286 2024-11-11T18:14:24,116 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:24,116 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:24,116 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/0786423d220c4037ae443cfba0dfa07b is 50, key is test_row_0/A:col10/1731348863615/Put/seqid=0 2024-11-11T18:14:24,117 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/e480089d27fd468b8abaaacb8eaf9f45 is 50, key is test_row_0/B:col10/1731348863615/Put/seqid=0 2024-11-11T18:14:24,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742109_1285 (size=12104) 2024-11-11T18:14:24,146 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/0786423d220c4037ae443cfba0dfa07b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/0786423d220c4037ae443cfba0dfa07b 2024-11-11T18:14:24,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742110_1286 (size=12104) 2024-11-11T18:14:24,158 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into 0786423d220c4037ae443cfba0dfa07b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:24,158 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:24,158 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=13, startTime=1731348864101; duration=0sec 2024-11-11T18:14:24,159 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:24,159 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:24,159 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:24,160 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:24,160 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:24,160 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,160 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/29dda5da037c4375993a5256a342bd5f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/1da070f975664cef9954c32c54bf95de, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/189abcc1ae124ca5ad105f8aa615a62c] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=35.2 K 2024-11-11T18:14:24,160 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29dda5da037c4375993a5256a342bd5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731348862584 2024-11-11T18:14:24,161 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da070f975664cef9954c32c54bf95de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731348862640 2024-11-11T18:14:24,161 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 189abcc1ae124ca5ad105f8aa615a62c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348863290 2024-11-11T18:14:24,170 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:24,171 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/fa70f6657d2244248c4595121b32647c is 50, key is test_row_0/C:col10/1731348863615/Put/seqid=0 2024-11-11T18:14:24,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T18:14:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,178 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T18:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/20daec4e441848569b9faac278bd4428 is 50, key is test_row_0/A:col10/1731348863673/Put/seqid=0 2024-11-11T18:14:24,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742111_1287 (size=12104) 2024-11-11T18:14:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742112_1288 (size=12001) 2024-11-11T18:14:24,227 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/20daec4e441848569b9faac278bd4428 2024-11-11T18:14:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/15e6b7a804394174aa7b8c9e0c684103 is 50, key is test_row_0/B:col10/1731348863673/Put/seqid=0 2024-11-11T18:14:24,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742113_1289 (size=12001) 2024-11-11T18:14:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:24,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348924319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348924325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348924329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348924329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348924329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T18:14:24,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348924435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,550 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/e480089d27fd468b8abaaacb8eaf9f45 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e480089d27fd468b8abaaacb8eaf9f45 2024-11-11T18:14:24,555 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into e480089d27fd468b8abaaacb8eaf9f45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:24,556 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:24,556 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=13, startTime=1731348864101; duration=0sec 2024-11-11T18:14:24,556 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:24,556 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:24,626 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/fa70f6657d2244248c4595121b32647c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fa70f6657d2244248c4595121b32647c 2024-11-11T18:14:24,631 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into fa70f6657d2244248c4595121b32647c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:24,632 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:24,632 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=13, startTime=1731348864101; duration=0sec 2024-11-11T18:14:24,632 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:24,632 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:24,647 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/15e6b7a804394174aa7b8c9e0c684103 2024-11-11T18:14:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348924645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/db72e7e5b7514935b5c637cf11fb791c is 50, key is test_row_0/C:col10/1731348863673/Put/seqid=0 2024-11-11T18:14:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742114_1290 (size=12001) 2024-11-11T18:14:24,667 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/db72e7e5b7514935b5c637cf11fb791c 2024-11-11T18:14:24,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/20daec4e441848569b9faac278bd4428 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/20daec4e441848569b9faac278bd4428 2024-11-11T18:14:24,676 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/20daec4e441848569b9faac278bd4428, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:14:24,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/15e6b7a804394174aa7b8c9e0c684103 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/15e6b7a804394174aa7b8c9e0c684103 2024-11-11T18:14:24,681 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/15e6b7a804394174aa7b8c9e0c684103, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:14:24,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/db72e7e5b7514935b5c637cf11fb791c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/db72e7e5b7514935b5c637cf11fb791c 2024-11-11T18:14:24,688 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/db72e7e5b7514935b5c637cf11fb791c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:14:24,689 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 7ac34b3c9a264f2c66d52e98e093bf68 in 511ms, sequenceid=96, compaction requested=false 2024-11-11T18:14:24,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:24,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:24,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T18:14:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T18:14:24,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T18:14:24,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 974 msec 2024-11-11T18:14:24,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 981 msec 2024-11-11T18:14:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:24,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:24,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:24,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:24,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:24,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/d55d787c7e284a57aa3f5f23c036f2fa is 50, key is test_row_0/A:col10/1731348864835/Put/seqid=0 2024-11-11T18:14:24,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742115_1291 (size=14341) 2024-11-11T18:14:24,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/d55d787c7e284a57aa3f5f23c036f2fa 2024-11-11T18:14:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T18:14:24,854 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:24,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T18:14:24,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:24,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:24,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:24,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/589cd94435904357a24831ae28f9dd38 is 50, key is test_row_0/B:col10/1731348864835/Put/seqid=0 2024-11-11T18:14:24,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348864851, max timestamp=1731348864874Num uniques 859; 2024-11-11T18:14:24,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348864851, max timestamp=1731348864874Num uniques 859; 2024-11-11T18:14:24,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:24,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:14:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348924871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731348864851, max timestamp=1731348864874Num uniques 859; 2024-11-11T18:14:24,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T18:14:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742116_1292 (size=12001) 2024-11-11T18:14:24,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/589cd94435904357a24831ae28f9dd38 2024-11-11T18:14:24,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/bd4e940b1daa448c96f1f05e458f21ee is 50, key is test_row_0/C:col10/1731348864835/Put/seqid=0 2024-11-11T18:14:24,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742117_1293 (size=12001) 2024-11-11T18:14:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348924913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348924914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348924921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348924954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348924984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T18:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348925024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348925025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348925034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T18:14:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348925194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348925234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348925234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348925245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/bd4e940b1daa448c96f1f05e458f21ee 2024-11-11T18:14:25,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T18:14:25,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/d55d787c7e284a57aa3f5f23c036f2fa as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d55d787c7e284a57aa3f5f23c036f2fa 2024-11-11T18:14:25,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d55d787c7e284a57aa3f5f23c036f2fa, entries=200, sequenceid=109, filesize=14.0 K 2024-11-11T18:14:25,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/589cd94435904357a24831ae28f9dd38 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/589cd94435904357a24831ae28f9dd38 2024-11-11T18:14:25,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/589cd94435904357a24831ae28f9dd38, entries=150, sequenceid=109, filesize=11.7 K 2024-11-11T18:14:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/bd4e940b1daa448c96f1f05e458f21ee as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/bd4e940b1daa448c96f1f05e458f21ee 2024-11-11T18:14:25,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/bd4e940b1daa448c96f1f05e458f21ee, entries=150, sequenceid=109, filesize=11.7 K 2024-11-11T18:14:25,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.84 KB/127835 for 7ac34b3c9a264f2c66d52e98e093bf68 in 511ms, sequenceid=109, compaction requested=true 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:25,347 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:25,347 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:25,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:25,353 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:25,353 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:25,353 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,353 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/0786423d220c4037ae443cfba0dfa07b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/20daec4e441848569b9faac278bd4428, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d55d787c7e284a57aa3f5f23c036f2fa] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=37.5 K 2024-11-11T18:14:25,354 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:25,354 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:25,354 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,354 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e480089d27fd468b8abaaacb8eaf9f45, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/15e6b7a804394174aa7b8c9e0c684103, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/589cd94435904357a24831ae28f9dd38] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=35.3 K 2024-11-11T18:14:25,354 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0786423d220c4037ae443cfba0dfa07b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348863290 2024-11-11T18:14:25,355 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e480089d27fd468b8abaaacb8eaf9f45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348863290 2024-11-11T18:14:25,355 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20daec4e441848569b9faac278bd4428, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348863673 2024-11-11T18:14:25,355 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 15e6b7a804394174aa7b8c9e0c684103, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348863673 2024-11-11T18:14:25,355 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55d787c7e284a57aa3f5f23c036f2fa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731348864298 2024-11-11T18:14:25,355 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 589cd94435904357a24831ae28f9dd38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731348864298 2024-11-11T18:14:25,370 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:25,371 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/4b463ad42e7d47f699c9ad2b8af31bd6 is 50, key is test_row_0/B:col10/1731348864835/Put/seqid=0 2024-11-11T18:14:25,374 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:25,375 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/b557477df0f643a9a419a9792dc3b966 is 50, key is test_row_0/A:col10/1731348864835/Put/seqid=0 2024-11-11T18:14:25,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742118_1294 (size=12207) 2024-11-11T18:14:25,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742119_1295 (size=12207) 2024-11-11T18:14:25,443 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/b557477df0f643a9a419a9792dc3b966 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/b557477df0f643a9a419a9792dc3b966 2024-11-11T18:14:25,455 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into b557477df0f643a9a419a9792dc3b966(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:25,455 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:25,455 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=13, startTime=1731348865347; duration=0sec 2024-11-11T18:14:25,455 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:25,456 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:25,456 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:25,457 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:25,457 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:25,457 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,457 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fa70f6657d2244248c4595121b32647c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/db72e7e5b7514935b5c637cf11fb791c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/bd4e940b1daa448c96f1f05e458f21ee] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=35.3 K 2024-11-11T18:14:25,458 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa70f6657d2244248c4595121b32647c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348863290 2024-11-11T18:14:25,459 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting db72e7e5b7514935b5c637cf11fb791c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348863673 2024-11-11T18:14:25,460 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4e940b1daa448c96f1f05e458f21ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731348864298 2024-11-11T18:14:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:25,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-11-11T18:14:25,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:25,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:25,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:25,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:25,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T18:14:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,475 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:25,476 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/589026c166564710afbce7f82d9b1053 is 50, key is test_row_0/C:col10/1731348864835/Put/seqid=0 2024-11-11T18:14:25,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/225727f47aa541e09b87df0ff37190fc is 50, key is test_row_0/A:col10/1731348864919/Put/seqid=0 2024-11-11T18:14:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:25,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348925509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348925517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742120_1296 (size=12207) 2024-11-11T18:14:25,546 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/589026c166564710afbce7f82d9b1053 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/589026c166564710afbce7f82d9b1053 2024-11-11T18:14:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348925544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348925544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742121_1297 (size=14541) 2024-11-11T18:14:25,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/225727f47aa541e09b87df0ff37190fc 2024-11-11T18:14:25,553 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into 589026c166564710afbce7f82d9b1053(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:25,553 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:25,553 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=13, startTime=1731348865347; duration=0sec 2024-11-11T18:14:25,554 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:25,554 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348925555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/799db3f10eb04716b5867b1abd94046e is 50, key is test_row_0/B:col10/1731348864919/Put/seqid=0 2024-11-11T18:14:25,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742122_1298 (size=12151) 2024-11-11T18:14:25,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/799db3f10eb04716b5867b1abd94046e 2024-11-11T18:14:25,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/dd8f04422a44432ca8f34c72dd433323 is 50, key is test_row_0/C:col10/1731348864919/Put/seqid=0 2024-11-11T18:14:25,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T18:14:25,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742123_1299 (size=12151) 2024-11-11T18:14:25,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:25,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:25,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348925625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/dd8f04422a44432ca8f34c72dd433323 2024-11-11T18:14:25,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/225727f47aa541e09b87df0ff37190fc as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/225727f47aa541e09b87df0ff37190fc 2024-11-11T18:14:25,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348925634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/225727f47aa541e09b87df0ff37190fc, entries=200, sequenceid=146, filesize=14.2 K 2024-11-11T18:14:25,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/799db3f10eb04716b5867b1abd94046e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/799db3f10eb04716b5867b1abd94046e 2024-11-11T18:14:25,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/799db3f10eb04716b5867b1abd94046e, entries=150, sequenceid=146, filesize=11.9 K 2024-11-11T18:14:25,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/dd8f04422a44432ca8f34c72dd433323 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/dd8f04422a44432ca8f34c72dd433323 2024-11-11T18:14:25,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/dd8f04422a44432ca8f34c72dd433323, entries=150, sequenceid=146, filesize=11.9 K 2024-11-11T18:14:25,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for 7ac34b3c9a264f2c66d52e98e093bf68 in 188ms, sequenceid=146, compaction requested=false 2024-11-11T18:14:25,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:25,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T18:14:25,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:25,780 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:25,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:25,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:25,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:25,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/7071b228c60c433ea8aee4d3e6b620b5 is 50, key is test_row_0/A:col10/1731348865507/Put/seqid=0 2024-11-11T18:14:25,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742124_1300 (size=12151) 2024-11-11T18:14:25,820 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/4b463ad42e7d47f699c9ad2b8af31bd6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4b463ad42e7d47f699c9ad2b8af31bd6 2024-11-11T18:14:25,826 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into 4b463ad42e7d47f699c9ad2b8af31bd6(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:25,826 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:25,826 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=13, startTime=1731348865347; duration=0sec 2024-11-11T18:14:25,826 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:25,826 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:25,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:25,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348865862, max timestamp=1731348865890Num uniques 859; 2024-11-11T18:14:25,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:25,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348865862, max timestamp=1731348865890Num uniques 859; 2024-11-11T18:14:25,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:25,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348865862, max timestamp=1731348865890Num uniques 859; 2024-11-11T18:14:25,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348925960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348925966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:25,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348926054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348926054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348926075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348926075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348926084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,199 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/7071b228c60c433ea8aee4d3e6b620b5 2024-11-11T18:14:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ae1a4c41db29418a8c40534ab3921146 is 50, key is test_row_0/B:col10/1731348865507/Put/seqid=0 2024-11-11T18:14:26,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742125_1301 (size=12151) 2024-11-11T18:14:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348926285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348926294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348926594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348926604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:26,615 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:14:26,615 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ae1a4c41db29418a8c40534ab3921146 2024-11-11T18:14:26,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/be22853979364503ab92a68e25aa1743 is 50, key is test_row_0/C:col10/1731348865507/Put/seqid=0 2024-11-11T18:14:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742126_1302 (size=12151) 2024-11-11T18:14:27,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:27,032 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/be22853979364503ab92a68e25aa1743 2024-11-11T18:14:27,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/7071b228c60c433ea8aee4d3e6b620b5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7071b228c60c433ea8aee4d3e6b620b5 2024-11-11T18:14:27,041 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7071b228c60c433ea8aee4d3e6b620b5, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T18:14:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ae1a4c41db29418a8c40534ab3921146 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ae1a4c41db29418a8c40534ab3921146 2024-11-11T18:14:27,046 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ae1a4c41db29418a8c40534ab3921146, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T18:14:27,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/be22853979364503ab92a68e25aa1743 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/be22853979364503ab92a68e25aa1743 2024-11-11T18:14:27,052 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/be22853979364503ab92a68e25aa1743, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T18:14:27,053 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1272ms, sequenceid=157, compaction requested=true 2024-11-11T18:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T18:14:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T18:14:27,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T18:14:27,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-11T18:14:27,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2010 sec 2024-11-11T18:14:27,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:27,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T18:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:27,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:14:27,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:27,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:14:27,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:27,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:14:27,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/1907f6b45a0f442d8d1b533d4296f0e8 is 50, key is test_row_0/A:col10/1731348865958/Put/seqid=0 2024-11-11T18:14:27,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742127_1303 (size=14541) 2024-11-11T18:14:27,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348927084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348927086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,091 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348927090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348927104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348927114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348927194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348927204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348927404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348927414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/1907f6b45a0f442d8d1b533d4296f0e8 2024-11-11T18:14:27,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/4e413ff03fdd48568fa7154caf4c911d is 50, key is test_row_0/B:col10/1731348865958/Put/seqid=0 2024-11-11T18:14:27,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742128_1304 (size=12151) 2024-11-11T18:14:27,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348927714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348927725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:27,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/4e413ff03fdd48568fa7154caf4c911d 2024-11-11T18:14:27,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/77e16a9f8a1947b881c7faf042aea08b is 50, key is test_row_0/C:col10/1731348865958/Put/seqid=0 2024-11-11T18:14:27,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742129_1305 (size=12151) 2024-11-11T18:14:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348928115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:28,119 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348928124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:28,129 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348928236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348928236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:28,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/77e16a9f8a1947b881c7faf042aea08b 2024-11-11T18:14:28,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/1907f6b45a0f442d8d1b533d4296f0e8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1907f6b45a0f442d8d1b533d4296f0e8 2024-11-11T18:14:28,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1907f6b45a0f442d8d1b533d4296f0e8, entries=200, sequenceid=196, filesize=14.2 K 2024-11-11T18:14:28,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/4e413ff03fdd48568fa7154caf4c911d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4e413ff03fdd48568fa7154caf4c911d 2024-11-11T18:14:28,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4e413ff03fdd48568fa7154caf4c911d, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T18:14:28,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/77e16a9f8a1947b881c7faf042aea08b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/77e16a9f8a1947b881c7faf042aea08b 2024-11-11T18:14:28,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/77e16a9f8a1947b881c7faf042aea08b, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T18:14:28,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1260ms, sequenceid=196, compaction requested=true 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:28,330 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:28,330 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:28,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:28,335 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:28,335 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:28,335 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:28,335 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:28,335 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/b557477df0f643a9a419a9792dc3b966, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/225727f47aa541e09b87df0ff37190fc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7071b228c60c433ea8aee4d3e6b620b5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1907f6b45a0f442d8d1b533d4296f0e8] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=52.2 K 2024-11-11T18:14:28,336 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:28,336 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:28,336 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4b463ad42e7d47f699c9ad2b8af31bd6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/799db3f10eb04716b5867b1abd94046e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ae1a4c41db29418a8c40534ab3921146, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4e413ff03fdd48568fa7154caf4c911d] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=47.5 K 2024-11-11T18:14:28,336 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b463ad42e7d47f699c9ad2b8af31bd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731348864298 2024-11-11T18:14:28,336 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b557477df0f643a9a419a9792dc3b966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731348864298 2024-11-11T18:14:28,337 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 799db3f10eb04716b5867b1abd94046e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731348864913 2024-11-11T18:14:28,337 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 225727f47aa541e09b87df0ff37190fc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731348864909 2024-11-11T18:14:28,337 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1a4c41db29418a8c40534ab3921146, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731348865505 2024-11-11T18:14:28,337 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7071b228c60c433ea8aee4d3e6b620b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731348865505 2024-11-11T18:14:28,337 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e413ff03fdd48568fa7154caf4c911d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731348865958 2024-11-11T18:14:28,337 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1907f6b45a0f442d8d1b533d4296f0e8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731348865956 2024-11-11T18:14:28,347 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:28,347 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:28,347 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/93115ea0e85a4285bba1541413a358f7 is 50, key is test_row_0/B:col10/1731348865958/Put/seqid=0 2024-11-11T18:14:28,347 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/809506016b214db188bd8d53eb6e5d6b is 50, key is test_row_0/A:col10/1731348865958/Put/seqid=0 2024-11-11T18:14:28,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742130_1306 (size=12493) 2024-11-11T18:14:28,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742131_1307 (size=12493) 2024-11-11T18:14:28,358 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/809506016b214db188bd8d53eb6e5d6b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/809506016b214db188bd8d53eb6e5d6b 2024-11-11T18:14:28,363 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into 809506016b214db188bd8d53eb6e5d6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:28,363 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:28,363 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=12, startTime=1731348868330; duration=0sec 2024-11-11T18:14:28,363 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:28,363 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:28,363 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:28,365 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:28,365 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:28,365 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:28,365 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/589026c166564710afbce7f82d9b1053, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/dd8f04422a44432ca8f34c72dd433323, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/be22853979364503ab92a68e25aa1743, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/77e16a9f8a1947b881c7faf042aea08b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=47.5 K 2024-11-11T18:14:28,366 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 589026c166564710afbce7f82d9b1053, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731348864298 2024-11-11T18:14:28,366 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8f04422a44432ca8f34c72dd433323, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731348864913 2024-11-11T18:14:28,366 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting be22853979364503ab92a68e25aa1743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731348865505 2024-11-11T18:14:28,366 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77e16a9f8a1947b881c7faf042aea08b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731348865958 2024-11-11T18:14:28,375 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:28,376 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd is 50, key is test_row_0/C:col10/1731348865958/Put/seqid=0 2024-11-11T18:14:28,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742132_1308 (size=12493) 2024-11-11T18:14:28,385 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd 2024-11-11T18:14:28,394 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into 7aa23b7ba3ab4a7da83a1b9750e9a5fd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:28,394 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:28,394 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=12, startTime=1731348868330; duration=0sec 2024-11-11T18:14:28,394 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:28,394 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:28,758 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/93115ea0e85a4285bba1541413a358f7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/93115ea0e85a4285bba1541413a358f7 2024-11-11T18:14:28,763 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into 93115ea0e85a4285bba1541413a358f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:28,763 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:28,763 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=12, startTime=1731348868330; duration=0sec 2024-11-11T18:14:28,763 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:28,763 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T18:14:29,024 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:29,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T18:14:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:29,027 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:29,027 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:29,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:29,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:29,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/01803201dddf41a5b0acb7770edc56e4 is 50, key is test_row_0/A:col10/1731348869106/Put/seqid=0 2024-11-11T18:14:29,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742133_1309 (size=12151) 2024-11-11T18:14:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:29,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:29,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348869149, max timestamp=1731348869192Num uniques 859; 2024-11-11T18:14:29,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348869149, max timestamp=1731348869192Num uniques 859; 2024-11-11T18:14:29,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348869149, max timestamp=1731348869192Num uniques 859; 2024-11-11T18:14:29,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:29,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:29,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:29,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731348869243, max timestamp=1731348869265Num uniques 1018; 2024-11-11T18:14:29,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731348869243, max timestamp=1731348869265Num uniques 1018; 2024-11-11T18:14:29,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T18:14:29,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-11T18:14:29,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348869258, max timestamp=1731348869265Num uniques 1018; 2024-11-11T18:14:29,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:14:29,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348929328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348929329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348929330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:29,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348929445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348929445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348929445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:29,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:29,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/01803201dddf41a5b0acb7770edc56e4 2024-11-11T18:14:29,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/8cc2fd1a194f4a169cbd9b0b531db2c0 is 50, key is test_row_0/B:col10/1731348869106/Put/seqid=0 2024-11-11T18:14:29,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742134_1310 (size=12151) 2024-11-11T18:14:29,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:29,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:29,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:29,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348929655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348929655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348929655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/8cc2fd1a194f4a169cbd9b0b531db2c0 2024-11-11T18:14:29,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/2490bc4ea5ce437cafc08393acc31e1a is 50, key is test_row_0/C:col10/1731348869106/Put/seqid=0 2024-11-11T18:14:29,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742135_1311 (size=12151) 2024-11-11T18:14:29,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:29,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:29,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:29,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348929964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348929964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348929965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:30,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:30,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:30,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:30,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:30,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:30,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348930134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,139 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348930144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,149 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:30,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:30,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:30,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/2490bc4ea5ce437cafc08393acc31e1a 2024-11-11T18:14:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/01803201dddf41a5b0acb7770edc56e4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/01803201dddf41a5b0acb7770edc56e4 2024-11-11T18:14:30,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/01803201dddf41a5b0acb7770edc56e4, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T18:14:30,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/8cc2fd1a194f4a169cbd9b0b531db2c0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/8cc2fd1a194f4a169cbd9b0b531db2c0 2024-11-11T18:14:30,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/8cc2fd1a194f4a169cbd9b0b531db2c0, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T18:14:30,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/2490bc4ea5ce437cafc08393acc31e1a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/2490bc4ea5ce437cafc08393acc31e1a 2024-11-11T18:14:30,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/2490bc4ea5ce437cafc08393acc31e1a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T18:14:30,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1256ms, sequenceid=209, compaction requested=false 2024-11-11T18:14:30,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:30,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T18:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:30,406 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=136.42 KB heapSize=384.53 KB 2024-11-11T18:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:30,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/3ed72a8bf9744b52a617965ca6b9380e is 50, key is test_row_0/A:col10/1731348869326/Put/seqid=0 2024-11-11T18:14:30,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742136_1312 (size=12251) 2024-11-11T18:14:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:30,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:30,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348930489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348930490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348930492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348930604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348930604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348930605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348930814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:30,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348930814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348930814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:30,824 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/3ed72a8bf9744b52a617965ca6b9380e 2024-11-11T18:14:30,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/0d644b0c56db4da8825d800f659ae4b9 is 50, key is test_row_0/B:col10/1731348869326/Put/seqid=0 2024-11-11T18:14:30,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742137_1313 (size=12251) 2024-11-11T18:14:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348931124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348931125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348931125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:31,236 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/0d644b0c56db4da8825d800f659ae4b9 2024-11-11T18:14:31,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/6d4f5191334e436bbc62701b560c8959 is 50, key is test_row_0/C:col10/1731348869326/Put/seqid=0 2024-11-11T18:14:31,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742138_1314 (size=12251) 2024-11-11T18:14:31,248 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/6d4f5191334e436bbc62701b560c8959 2024-11-11T18:14:31,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/3ed72a8bf9744b52a617965ca6b9380e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3ed72a8bf9744b52a617965ca6b9380e 2024-11-11T18:14:31,258 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3ed72a8bf9744b52a617965ca6b9380e, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T18:14:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/0d644b0c56db4da8825d800f659ae4b9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/0d644b0c56db4da8825d800f659ae4b9 2024-11-11T18:14:31,262 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/0d644b0c56db4da8825d800f659ae4b9, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T18:14:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/6d4f5191334e436bbc62701b560c8959 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6d4f5191334e436bbc62701b560c8959 2024-11-11T18:14:31,266 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6d4f5191334e436bbc62701b560c8959, entries=150, sequenceid=262, filesize=12.0 K 2024-11-11T18:14:31,267 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~384.48 KB/393712, currentSize=46.96 KB/48090 for 7ac34b3c9a264f2c66d52e98e093bf68 in 861ms, sequenceid=262, compaction requested=true 2024-11-11T18:14:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:31,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:31,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T18:14:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T18:14:31,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T18:14:31,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2410 sec 2024-11-11T18:14:31,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.2460 sec 2024-11-11T18:14:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:31,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:14:31,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:31,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:31,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:31,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:31,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/a92d7afcd5ed4451a251fb7956cdb08d is 50, key is test_row_0/A:col10/1731348871636/Put/seqid=0 2024-11-11T18:14:31,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742139_1315 (size=17181) 2024-11-11T18:14:31,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/a92d7afcd5ed4451a251fb7956cdb08d 2024-11-11T18:14:31,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/2917a9c385b544d096b56d39a9dbbc8e is 50, key is test_row_0/B:col10/1731348871636/Put/seqid=0 2024-11-11T18:14:31,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742140_1316 (size=12301) 2024-11-11T18:14:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348931673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348931674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348931678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348931784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348931784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348931785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348931994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348931994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348931995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/2917a9c385b544d096b56d39a9dbbc8e 2024-11-11T18:14:32,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/feb709b2b7b94c2a88df952bf79d0dcc is 50, key is test_row_0/C:col10/1731348871636/Put/seqid=0 2024-11-11T18:14:32,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742141_1317 (size=12301) 2024-11-11T18:14:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348932304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348932304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348932304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/feb709b2b7b94c2a88df952bf79d0dcc 2024-11-11T18:14:32,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/a92d7afcd5ed4451a251fb7956cdb08d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/a92d7afcd5ed4451a251fb7956cdb08d 2024-11-11T18:14:32,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/a92d7afcd5ed4451a251fb7956cdb08d, entries=250, sequenceid=274, filesize=16.8 K 2024-11-11T18:14:32,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/2917a9c385b544d096b56d39a9dbbc8e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/2917a9c385b544d096b56d39a9dbbc8e 2024-11-11T18:14:32,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/2917a9c385b544d096b56d39a9dbbc8e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T18:14:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/feb709b2b7b94c2a88df952bf79d0dcc as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/feb709b2b7b94c2a88df952bf79d0dcc 2024-11-11T18:14:32,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/feb709b2b7b94c2a88df952bf79d0dcc, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T18:14:32,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7ac34b3c9a264f2c66d52e98e093bf68 in 859ms, sequenceid=274, compaction requested=true 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:32,497 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T18:14:32,497 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:32,498 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54076 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:32,498 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:32,498 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49196 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:32,498 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:32,498 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:32,499 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:32,499 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/809506016b214db188bd8d53eb6e5d6b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/01803201dddf41a5b0acb7770edc56e4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3ed72a8bf9744b52a617965ca6b9380e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/a92d7afcd5ed4451a251fb7956cdb08d] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=52.8 K 2024-11-11T18:14:32,499 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/93115ea0e85a4285bba1541413a358f7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/8cc2fd1a194f4a169cbd9b0b531db2c0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/0d644b0c56db4da8825d800f659ae4b9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/2917a9c385b544d096b56d39a9dbbc8e] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=48.0 K 2024-11-11T18:14:32,499 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 809506016b214db188bd8d53eb6e5d6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731348865958 2024-11-11T18:14:32,499 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 93115ea0e85a4285bba1541413a358f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731348865958 2024-11-11T18:14:32,500 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc2fd1a194f4a169cbd9b0b531db2c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731348867080 2024-11-11T18:14:32,500 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01803201dddf41a5b0acb7770edc56e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731348867080 2024-11-11T18:14:32,500 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d644b0c56db4da8825d800f659ae4b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731348869287 2024-11-11T18:14:32,500 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ed72a8bf9744b52a617965ca6b9380e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731348869287 2024-11-11T18:14:32,500 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2917a9c385b544d096b56d39a9dbbc8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348870476 2024-11-11T18:14:32,501 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92d7afcd5ed4451a251fb7956cdb08d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348870476 2024-11-11T18:14:32,509 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:32,510 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ac88276139824c09906be8e94acc9f33 is 50, key is test_row_0/B:col10/1731348871636/Put/seqid=0 2024-11-11T18:14:32,510 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#271 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:32,510 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/5d598449c5114d419af051ca9759e76d is 50, key is test_row_0/A:col10/1731348871636/Put/seqid=0 2024-11-11T18:14:32,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742143_1319 (size=12779) 2024-11-11T18:14:32,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742142_1318 (size=12779) 2024-11-11T18:14:32,526 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ac88276139824c09906be8e94acc9f33 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ac88276139824c09906be8e94acc9f33 2024-11-11T18:14:32,532 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into ac88276139824c09906be8e94acc9f33(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:32,532 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:32,534 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=12, startTime=1731348872497; duration=0sec 2024-11-11T18:14:32,534 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:32,534 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:32,534 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:32,535 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49196 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:32,535 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:32,535 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:32,535 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/2490bc4ea5ce437cafc08393acc31e1a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6d4f5191334e436bbc62701b560c8959, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/feb709b2b7b94c2a88df952bf79d0dcc] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=48.0 K 2024-11-11T18:14:32,536 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa23b7ba3ab4a7da83a1b9750e9a5fd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731348865958 2024-11-11T18:14:32,536 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2490bc4ea5ce437cafc08393acc31e1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731348867080 2024-11-11T18:14:32,537 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4f5191334e436bbc62701b560c8959, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731348869287 2024-11-11T18:14:32,537 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting feb709b2b7b94c2a88df952bf79d0dcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348870476 2024-11-11T18:14:32,551 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:32,552 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/eb930bb446ec4175a632b990ad4b2980 is 50, key is test_row_0/C:col10/1731348871636/Put/seqid=0 2024-11-11T18:14:32,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742144_1320 (size=12779) 2024-11-11T18:14:32,575 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/eb930bb446ec4175a632b990ad4b2980 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eb930bb446ec4175a632b990ad4b2980 2024-11-11T18:14:32,580 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into eb930bb446ec4175a632b990ad4b2980(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:32,580 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:32,580 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=12, startTime=1731348872497; duration=0sec 2024-11-11T18:14:32,580 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:32,580 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:32,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T18:14:32,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:32,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:32,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:32,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:32,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:32,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:32,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/7c81789bf75940deabd51943814efe06 is 50, key is test_row_0/A:col10/1731348871677/Put/seqid=0 2024-11-11T18:14:32,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742145_1321 (size=12301) 2024-11-11T18:14:32,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348932829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348932831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348932834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,923 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/5d598449c5114d419af051ca9759e76d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5d598449c5114d419af051ca9759e76d 2024-11-11T18:14:32,927 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into 5d598449c5114d419af051ca9759e76d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:32,927 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:32,927 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=12, startTime=1731348872497; duration=0sec 2024-11-11T18:14:32,928 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:32,928 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:32,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348932944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348932944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:32,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348932944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348933155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348933155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348933155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T18:14:33,184 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:33,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-11T18:14:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T18:14:33,187 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:33,188 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:33,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:33,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/7c81789bf75940deabd51943814efe06 2024-11-11T18:14:33,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/9d4275e6f8014228825db6e426df3aec is 50, key is test_row_0/B:col10/1731348871677/Put/seqid=0 2024-11-11T18:14:33,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742146_1322 (size=12301) 2024-11-11T18:14:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T18:14:33,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T18:14:33,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:33,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348933464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348933465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348933465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:33,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T18:14:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T18:14:33,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/9d4275e6f8014228825db6e426df3aec 2024-11-11T18:14:33,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/6022fe16ebfb474bb1c6a190f38eeb12 is 50, key is test_row_0/C:col10/1731348871677/Put/seqid=0 2024-11-11T18:14:33,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T18:14:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742147_1323 (size=12301) 2024-11-11T18:14:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:33,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/6022fe16ebfb474bb1c6a190f38eeb12 2024-11-11T18:14:33,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/7c81789bf75940deabd51943814efe06 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7c81789bf75940deabd51943814efe06 2024-11-11T18:14:33,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7c81789bf75940deabd51943814efe06, entries=150, sequenceid=297, filesize=12.0 K 2024-11-11T18:14:33,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/9d4275e6f8014228825db6e426df3aec as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/9d4275e6f8014228825db6e426df3aec 2024-11-11T18:14:33,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/9d4275e6f8014228825db6e426df3aec, entries=150, sequenceid=297, filesize=12.0 K 2024-11-11T18:14:33,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/6022fe16ebfb474bb1c6a190f38eeb12 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6022fe16ebfb474bb1c6a190f38eeb12 2024-11-11T18:14:33,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6022fe16ebfb474bb1c6a190f38eeb12, entries=150, sequenceid=297, filesize=12.0 K 2024-11-11T18:14:33,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 7ac34b3c9a264f2c66d52e98e093bf68 in 861ms, sequenceid=297, compaction requested=false 2024-11-11T18:14:33,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:33,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:33,807 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:33,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/770e09ca3c3941188ee09661a8cc6770 is 50, key is test_row_0/A:col10/1731348872832/Put/seqid=0 2024-11-11T18:14:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T18:14:33,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742148_1324 (size=12301) 2024-11-11T18:14:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:33,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348873988, max timestamp=1731348874030Num uniques 859; 2024-11-11T18:14:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348873988, max timestamp=1731348874030Num uniques 859; 2024-11-11T18:14:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348873988, max timestamp=1731348874030Num uniques 859; 2024-11-11T18:14:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:14:34,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348934065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348934066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348934067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44024 deadline: 1731348934154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,157 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348934175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44048 deadline: 1731348934184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,186 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 7ac34b3c9a264f2c66d52e98e093bf68 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:14:34,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,217 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/770e09ca3c3941188ee09661a8cc6770 2024-11-11T18:14:34,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/d8dccd5004f44cf3988c9bb08631b345 is 50, key is test_row_0/B:col10/1731348872832/Put/seqid=0 2024-11-11T18:14:34,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742149_1325 (size=12301) 2024-11-11T18:14:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T18:14:34,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348934384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348934384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348934384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,630 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/d8dccd5004f44cf3988c9bb08631b345 2024-11-11T18:14:34,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/c75bcb8d5fb845b98de374f20a687cc0 is 50, key is test_row_0/C:col10/1731348872832/Put/seqid=0 2024-11-11T18:14:34,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742150_1326 (size=12301) 2024-11-11T18:14:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348934694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348934694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348934694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,042 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/c75bcb8d5fb845b98de374f20a687cc0 2024-11-11T18:14:35,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/770e09ca3c3941188ee09661a8cc6770 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/770e09ca3c3941188ee09661a8cc6770 2024-11-11T18:14:35,051 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/770e09ca3c3941188ee09661a8cc6770, entries=150, sequenceid=309, filesize=12.0 K 2024-11-11T18:14:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/d8dccd5004f44cf3988c9bb08631b345 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/d8dccd5004f44cf3988c9bb08631b345 2024-11-11T18:14:35,055 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/d8dccd5004f44cf3988c9bb08631b345, entries=150, sequenceid=309, filesize=12.0 K 2024-11-11T18:14:35,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/c75bcb8d5fb845b98de374f20a687cc0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/c75bcb8d5fb845b98de374f20a687cc0 2024-11-11T18:14:35,059 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/c75bcb8d5fb845b98de374f20a687cc0, entries=150, sequenceid=309, filesize=12.0 K 2024-11-11T18:14:35,060 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1253ms, sequenceid=309, compaction requested=true 2024-11-11T18:14:35,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-11T18:14:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-11T18:14:35,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T18:14:35,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8740 sec 2024-11-11T18:14:35,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8790 sec 2024-11-11T18:14:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7683, totalHeapSize=17712, min timestamp=1731348874064, max timestamp=1731348875208Num uniques 1018; 2024-11-11T18:14:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46613, new segment data size=7683, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:14:35,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=109.58 KB heapSize=312.13 KB 2024-11-11T18:14:35,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:35,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:35,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:35,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7683, totalHeapSize=17712, min timestamp=1731348874064, max timestamp=1731348875208Num uniques 1018; 2024-11-11T18:14:35,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:35,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:35,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46613, new segment data size=7683, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:14:35,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:14:35,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/51e7d8563c2047dfb205d64ac6a193df is 50, key is test_row_0/A:col10/1731348875207/Put/seqid=0 2024-11-11T18:14:35,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742151_1327 (size=12301) 2024-11-11T18:14:35,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348935246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348935246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348935248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T18:14:35,345 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:35,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-11T18:14:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T18:14:35,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:35,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:35,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348935354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348935354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348935364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T18:14:35,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T18:14:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348935564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348935564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348935574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/51e7d8563c2047dfb205d64ac6a193df 2024-11-11T18:14:35,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/7af4b02d476e4a02a9e99cf7e080901b is 50, key is test_row_0/B:col10/1731348875207/Put/seqid=0 2024-11-11T18:14:35,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742152_1328 (size=12301) 2024-11-11T18:14:35,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T18:14:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T18:14:35,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T18:14:35,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:35,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348935874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348935874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348935884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:35,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T18:14:35,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:35,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:35,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T18:14:36,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/7af4b02d476e4a02a9e99cf7e080901b 2024-11-11T18:14:36,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/73f66144961a4df6affbea25c0abb973 is 50, key is test_row_0/C:col10/1731348875207/Put/seqid=0 2024-11-11T18:14:36,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742153_1329 (size=12301) 2024-11-11T18:14:36,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/73f66144961a4df6affbea25c0abb973 2024-11-11T18:14:36,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/51e7d8563c2047dfb205d64ac6a193df as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/51e7d8563c2047dfb205d64ac6a193df 2024-11-11T18:14:36,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/51e7d8563c2047dfb205d64ac6a193df, entries=150, sequenceid=348, filesize=12.0 K 2024-11-11T18:14:36,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/7af4b02d476e4a02a9e99cf7e080901b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/7af4b02d476e4a02a9e99cf7e080901b 2024-11-11T18:14:36,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/7af4b02d476e4a02a9e99cf7e080901b, entries=150, sequenceid=348, filesize=12.0 K 2024-11-11T18:14:36,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/73f66144961a4df6affbea25c0abb973 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73f66144961a4df6affbea25c0abb973 2024-11-11T18:14:36,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73f66144961a4df6affbea25c0abb973, entries=150, sequenceid=348, filesize=12.0 K 2024-11-11T18:14:36,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for 7ac34b3c9a264f2c66d52e98e093bf68 in 883ms, sequenceid=348, compaction requested=true 2024-11-11T18:14:36,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:36,097 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:36,097 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:36,099 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:36,099 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:36,099 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:36,099 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:36,099 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:36,099 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:36,099 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5d598449c5114d419af051ca9759e76d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7c81789bf75940deabd51943814efe06, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/770e09ca3c3941188ee09661a8cc6770, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/51e7d8563c2047dfb205d64ac6a193df] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=48.5 K 2024-11-11T18:14:36,099 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ac88276139824c09906be8e94acc9f33, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/9d4275e6f8014228825db6e426df3aec, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/d8dccd5004f44cf3988c9bb08631b345, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/7af4b02d476e4a02a9e99cf7e080901b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=48.5 K 2024-11-11T18:14:36,100 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d598449c5114d419af051ca9759e76d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348870476 2024-11-11T18:14:36,100 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ac88276139824c09906be8e94acc9f33, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348870476 2024-11-11T18:14:36,101 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c81789bf75940deabd51943814efe06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731348871673 2024-11-11T18:14:36,101 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4275e6f8014228825db6e426df3aec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731348871673 2024-11-11T18:14:36,101 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770e09ca3c3941188ee09661a8cc6770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731348872827 2024-11-11T18:14:36,101 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d8dccd5004f44cf3988c9bb08631b345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731348872827 2024-11-11T18:14:36,101 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e7d8563c2047dfb205d64ac6a193df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731348875207 2024-11-11T18:14:36,102 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af4b02d476e4a02a9e99cf7e080901b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731348875207 2024-11-11T18:14:36,113 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:36,114 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/d8843d02d3d844c7a089226e416b59f0 is 50, key is test_row_0/A:col10/1731348875207/Put/seqid=0 2024-11-11T18:14:36,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-11T18:14:36,117 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:36,118 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/744475d7809c449fa71450695dece9f7 is 50, key is test_row_0/B:col10/1731348875207/Put/seqid=0 2024-11-11T18:14:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:36,118 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T18:14:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:36,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/3105113da876420abc8ae2126a19855a is 50, key is test_row_0/A:col10/1731348875246/Put/seqid=0 2024-11-11T18:14:36,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742155_1331 (size=12915) 2024-11-11T18:14:36,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742154_1330 (size=12915) 2024-11-11T18:14:36,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742156_1332 (size=12301) 2024-11-11T18:14:36,223 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/3105113da876420abc8ae2126a19855a 2024-11-11T18:14:36,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/212fe7c445924b76824fc25522a63ea7 is 50, key is test_row_0/B:col10/1731348875246/Put/seqid=0 2024-11-11T18:14:36,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742157_1333 (size=12301) 2024-11-11T18:14:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:36,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:36,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348936427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348936429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348936433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T18:14:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348936544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348936544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348936544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,566 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/744475d7809c449fa71450695dece9f7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/744475d7809c449fa71450695dece9f7 2024-11-11T18:14:36,576 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into 744475d7809c449fa71450695dece9f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:36,576 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:36,576 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=12, startTime=1731348876097; duration=0sec 2024-11-11T18:14:36,576 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:36,576 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:36,576 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:14:36,577 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/d8843d02d3d844c7a089226e416b59f0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d8843d02d3d844c7a089226e416b59f0 2024-11-11T18:14:36,579 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:14:36,579 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:36,579 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:36,579 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eb930bb446ec4175a632b990ad4b2980, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6022fe16ebfb474bb1c6a190f38eeb12, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/c75bcb8d5fb845b98de374f20a687cc0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73f66144961a4df6affbea25c0abb973] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=48.5 K 2024-11-11T18:14:36,579 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting eb930bb446ec4175a632b990ad4b2980, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348870476 2024-11-11T18:14:36,581 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6022fe16ebfb474bb1c6a190f38eeb12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731348871673 2024-11-11T18:14:36,581 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c75bcb8d5fb845b98de374f20a687cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731348872827 2024-11-11T18:14:36,582 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f66144961a4df6affbea25c0abb973, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731348875207 2024-11-11T18:14:36,586 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into d8843d02d3d844c7a089226e416b59f0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:36,586 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:36,586 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=12, startTime=1731348876097; duration=0sec 2024-11-11T18:14:36,586 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:36,586 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:36,603 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:36,604 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/3629c464e61f43478b63cf567a852b70 is 50, key is test_row_0/C:col10/1731348875207/Put/seqid=0 2024-11-11T18:14:36,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742158_1334 (size=12915) 2024-11-11T18:14:36,628 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/3629c464e61f43478b63cf567a852b70 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/3629c464e61f43478b63cf567a852b70 2024-11-11T18:14:36,633 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into 3629c464e61f43478b63cf567a852b70(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:36,633 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:36,633 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=12, startTime=1731348876097; duration=0sec 2024-11-11T18:14:36,633 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:36,633 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:36,647 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/212fe7c445924b76824fc25522a63ea7 2024-11-11T18:14:36,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/fc0331e602cd4a5db570baae5c65f613 is 50, key is test_row_0/C:col10/1731348875246/Put/seqid=0 2024-11-11T18:14:36,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742159_1335 (size=12301) 2024-11-11T18:14:36,672 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/fc0331e602cd4a5db570baae5c65f613 2024-11-11T18:14:36,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/3105113da876420abc8ae2126a19855a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3105113da876420abc8ae2126a19855a 2024-11-11T18:14:36,687 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3105113da876420abc8ae2126a19855a, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T18:14:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/212fe7c445924b76824fc25522a63ea7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/212fe7c445924b76824fc25522a63ea7 2024-11-11T18:14:36,694 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/212fe7c445924b76824fc25522a63ea7, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T18:14:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/fc0331e602cd4a5db570baae5c65f613 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fc0331e602cd4a5db570baae5c65f613 2024-11-11T18:14:36,699 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fc0331e602cd4a5db570baae5c65f613, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T18:14:36,702 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7ac34b3c9a264f2c66d52e98e093bf68 in 584ms, sequenceid=367, compaction requested=false 2024-11-11T18:14:36,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:36,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:36,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-11T18:14:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-11T18:14:36,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T18:14:36,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3560 sec 2024-11-11T18:14:36,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3610 sec 2024-11-11T18:14:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:36,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T18:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:36,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:36,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/91df4725e6384d11bef93add18764e03 is 50, key is test_row_0/A:col10/1731348876757/Put/seqid=0 2024-11-11T18:14:36,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742160_1336 (size=14741) 2024-11-11T18:14:36,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/91df4725e6384d11bef93add18764e03 2024-11-11T18:14:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348936787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348936789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348936792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5d4ee92bdec44f1eb1ce9c399c55617e is 50, key is test_row_0/B:col10/1731348876757/Put/seqid=0 2024-11-11T18:14:36,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742161_1337 (size=12301) 2024-11-11T18:14:36,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348936904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348936905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:36,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348936905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:36,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348937114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348937115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348937125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5d4ee92bdec44f1eb1ce9c399c55617e 2024-11-11T18:14:37,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/639b57eb47f1473e82d6479b2e5e7199 is 50, key is test_row_0/C:col10/1731348876757/Put/seqid=0 2024-11-11T18:14:37,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742162_1338 (size=12301) 2024-11-11T18:14:37,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348937424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348937424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348937434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-11T18:14:37,495 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:37,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-11T18:14:37,498 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T18:14:37,498 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:37,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T18:14:37,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/639b57eb47f1473e82d6479b2e5e7199 2024-11-11T18:14:37,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/91df4725e6384d11bef93add18764e03 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/91df4725e6384d11bef93add18764e03 2024-11-11T18:14:37,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/91df4725e6384d11bef93add18764e03, entries=200, sequenceid=385, filesize=14.4 K 2024-11-11T18:14:37,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/5d4ee92bdec44f1eb1ce9c399c55617e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5d4ee92bdec44f1eb1ce9c399c55617e 2024-11-11T18:14:37,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T18:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:37,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5d4ee92bdec44f1eb1ce9c399c55617e, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T18:14:37,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:37,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:37,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/639b57eb47f1473e82d6479b2e5e7199 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/639b57eb47f1473e82d6479b2e5e7199 2024-11-11T18:14:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:37,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/639b57eb47f1473e82d6479b2e5e7199, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T18:14:37,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7ac34b3c9a264f2c66d52e98e093bf68 in 900ms, sequenceid=385, compaction requested=true 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:37,658 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:37,658 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:37,659 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:37,659 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:37,659 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:37,659 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/744475d7809c449fa71450695dece9f7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/212fe7c445924b76824fc25522a63ea7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5d4ee92bdec44f1eb1ce9c399c55617e] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.6 K 2024-11-11T18:14:37,660 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:37,660 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:37,660 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:37,660 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d8843d02d3d844c7a089226e416b59f0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3105113da876420abc8ae2126a19855a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/91df4725e6384d11bef93add18764e03] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=39.0 K 2024-11-11T18:14:37,660 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 744475d7809c449fa71450695dece9f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731348875207 2024-11-11T18:14:37,661 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8843d02d3d844c7a089226e416b59f0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731348875207 2024-11-11T18:14:37,661 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 212fe7c445924b76824fc25522a63ea7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731348875244 2024-11-11T18:14:37,661 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3105113da876420abc8ae2126a19855a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731348875244 2024-11-11T18:14:37,661 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4ee92bdec44f1eb1ce9c399c55617e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731348876427 2024-11-11T18:14:37,661 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91df4725e6384d11bef93add18764e03, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731348876398 2024-11-11T18:14:37,672 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:37,672 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/1aa538cd5bcb42b3bf0a6cab23c01c3b is 50, key is test_row_0/B:col10/1731348876757/Put/seqid=0 2024-11-11T18:14:37,673 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:37,673 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/2a7c1034e25d45d48445ec6c8b4036be is 50, key is test_row_0/A:col10/1731348876757/Put/seqid=0 2024-11-11T18:14:37,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742164_1340 (size=13017) 2024-11-11T18:14:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742163_1339 (size=13017) 2024-11-11T18:14:37,700 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/2a7c1034e25d45d48445ec6c8b4036be as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/2a7c1034e25d45d48445ec6c8b4036be 2024-11-11T18:14:37,708 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into 2a7c1034e25d45d48445ec6c8b4036be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:37,708 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:37,708 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=13, startTime=1731348877658; duration=0sec 2024-11-11T18:14:37,708 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:37,708 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:37,708 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:37,711 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:37,711 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:37,711 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:37,711 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/3629c464e61f43478b63cf567a852b70, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fc0331e602cd4a5db570baae5c65f613, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/639b57eb47f1473e82d6479b2e5e7199] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.6 K 2024-11-11T18:14:37,711 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3629c464e61f43478b63cf567a852b70, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731348875207 2024-11-11T18:14:37,712 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc0331e602cd4a5db570baae5c65f613, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731348875244 2024-11-11T18:14:37,712 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 639b57eb47f1473e82d6479b2e5e7199, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731348876427 2024-11-11T18:14:37,726 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:37,726 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/ff85b058d48647e8b65d7faadfea3628 is 50, key is test_row_0/C:col10/1731348876757/Put/seqid=0 2024-11-11T18:14:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742165_1341 (size=13017) 2024-11-11T18:14:37,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-11T18:14:37,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:37,804 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T18:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:37,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/5e89b89d8b3247fba13499a0a7fcbd71 is 50, key is test_row_0/A:col10/1731348876790/Put/seqid=0 2024-11-11T18:14:37,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742166_1342 (size=12301) 2024-11-11T18:14:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T18:14:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:37,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:37,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348937985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348937986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:37,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348937989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348938094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348938094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,098 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/1aa538cd5bcb42b3bf0a6cab23c01c3b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/1aa538cd5bcb42b3bf0a6cab23c01c3b 2024-11-11T18:14:38,102 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into 1aa538cd5bcb42b3bf0a6cab23c01c3b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:38,102 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:38,102 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=13, startTime=1731348877658; duration=0sec 2024-11-11T18:14:38,103 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:38,103 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:38,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348938104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T18:14:38,147 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/ff85b058d48647e8b65d7faadfea3628 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ff85b058d48647e8b65d7faadfea3628 2024-11-11T18:14:38,151 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into ff85b058d48647e8b65d7faadfea3628(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:38,151 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:38,152 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=13, startTime=1731348877658; duration=0sec 2024-11-11T18:14:38,152 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:38,152 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:38,214 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/5e89b89d8b3247fba13499a0a7fcbd71 2024-11-11T18:14:38,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c63d43472de04b388d0e30b8e2f3ad60 is 50, key is test_row_0/B:col10/1731348876790/Put/seqid=0 2024-11-11T18:14:38,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742167_1343 (size=12301) 2024-11-11T18:14:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348938304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348938304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348938314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348938614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348938615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,626 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c63d43472de04b388d0e30b8e2f3ad60 2024-11-11T18:14:38,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:38,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348938624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/cf344ed577e04c36973edaf720d51afe is 50, key is test_row_0/C:col10/1731348876790/Put/seqid=0 2024-11-11T18:14:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T18:14:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742168_1344 (size=12301) 2024-11-11T18:14:39,038 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/cf344ed577e04c36973edaf720d51afe 2024-11-11T18:14:39,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/5e89b89d8b3247fba13499a0a7fcbd71 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5e89b89d8b3247fba13499a0a7fcbd71 2024-11-11T18:14:39,053 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5e89b89d8b3247fba13499a0a7fcbd71, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T18:14:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c63d43472de04b388d0e30b8e2f3ad60 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c63d43472de04b388d0e30b8e2f3ad60 2024-11-11T18:14:39,058 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c63d43472de04b388d0e30b8e2f3ad60, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T18:14:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/cf344ed577e04c36973edaf720d51afe as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/cf344ed577e04c36973edaf720d51afe 2024-11-11T18:14:39,063 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/cf344ed577e04c36973edaf720d51afe, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T18:14:39,064 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1260ms, sequenceid=403, compaction requested=false 2024-11-11T18:14:39,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-11T18:14:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-11T18:14:39,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-11T18:14:39,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5680 sec 2024-11-11T18:14:39,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.5720 sec 2024-11-11T18:14:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:39,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T18:14:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:39,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/87bdc64f8ad1435f88214110fbe56010 is 50, key is test_row_0/A:col10/1731348879126/Put/seqid=0 2024-11-11T18:14:39,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742169_1345 (size=12301) 2024-11-11T18:14:39,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348939152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348939153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348939154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348939264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348939264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348939264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348939474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348939474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348939474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/87bdc64f8ad1435f88214110fbe56010 2024-11-11T18:14:39,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ff945e02152e4de890ccdf5691f228b0 is 50, key is test_row_0/B:col10/1731348879126/Put/seqid=0 2024-11-11T18:14:39,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742170_1346 (size=12301) 2024-11-11T18:14:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T18:14:39,645 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:39,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-11T18:14:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T18:14:39,647 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:39,648 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:39,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T18:14:39,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348939785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348939785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348939785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:39,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T18:14:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:39,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ff945e02152e4de890ccdf5691f228b0 2024-11-11T18:14:39,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T18:14:39,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:39,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:39,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:39,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/8b01589c83044f7fb0e39bd498c212b7 is 50, key is test_row_0/C:col10/1731348879126/Put/seqid=0 2024-11-11T18:14:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T18:14:39,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742171_1347 (size=12301) 2024-11-11T18:14:39,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/8b01589c83044f7fb0e39bd498c212b7 2024-11-11T18:14:39,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/87bdc64f8ad1435f88214110fbe56010 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/87bdc64f8ad1435f88214110fbe56010 2024-11-11T18:14:39,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/87bdc64f8ad1435f88214110fbe56010, entries=150, sequenceid=421, filesize=12.0 K 2024-11-11T18:14:39,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/ff945e02152e4de890ccdf5691f228b0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ff945e02152e4de890ccdf5691f228b0 2024-11-11T18:14:39,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ff945e02152e4de890ccdf5691f228b0, entries=150, sequenceid=421, filesize=12.0 K 2024-11-11T18:14:39,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/8b01589c83044f7fb0e39bd498c212b7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/8b01589c83044f7fb0e39bd498c212b7 2024-11-11T18:14:39,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/8b01589c83044f7fb0e39bd498c212b7, entries=150, sequenceid=421, filesize=12.0 K 2024-11-11T18:14:39,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7ac34b3c9a264f2c66d52e98e093bf68 in 856ms, sequenceid=421, compaction requested=true 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:39,984 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:39,984 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:39,985 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:39,985 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:39,985 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,985 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:39,985 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/2a7c1034e25d45d48445ec6c8b4036be, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5e89b89d8b3247fba13499a0a7fcbd71, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/87bdc64f8ad1435f88214110fbe56010] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.7 K 2024-11-11T18:14:39,985 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:39,985 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:39,985 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/1aa538cd5bcb42b3bf0a6cab23c01c3b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c63d43472de04b388d0e30b8e2f3ad60, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ff945e02152e4de890ccdf5691f228b0] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.7 K 2024-11-11T18:14:39,985 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a7c1034e25d45d48445ec6c8b4036be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731348876427 2024-11-11T18:14:39,986 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e89b89d8b3247fba13499a0a7fcbd71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731348876784 2024-11-11T18:14:39,986 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87bdc64f8ad1435f88214110fbe56010, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731348877984 2024-11-11T18:14:39,986 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aa538cd5bcb42b3bf0a6cab23c01c3b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731348876427 2024-11-11T18:14:39,987 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c63d43472de04b388d0e30b8e2f3ad60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731348876784 2024-11-11T18:14:39,987 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ff945e02152e4de890ccdf5691f228b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731348877984 2024-11-11T18:14:39,993 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:39,994 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/291fd6bb28da4c4f89c9323856d64b54 is 50, key is test_row_0/A:col10/1731348879126/Put/seqid=0 2024-11-11T18:14:39,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742172_1348 (size=13119) 2024-11-11T18:14:39,999 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:39,999 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c4a8597cbf1a414796127d8af872cab1 is 50, key is test_row_0/B:col10/1731348879126/Put/seqid=0 2024-11-11T18:14:40,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742173_1349 (size=13119) 2024-11-11T18:14:40,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-11T18:14:40,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:40,106 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T18:14:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:40,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/49f5ce81ba514109a5a8f40bce2c3c32 is 50, key is test_row_0/A:col10/1731348879144/Put/seqid=0 2024-11-11T18:14:40,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742174_1350 (size=12301) 2024-11-11T18:14:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T18:14:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:40,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348940315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348940316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348940317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,405 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/291fd6bb28da4c4f89c9323856d64b54 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/291fd6bb28da4c4f89c9323856d64b54 2024-11-11T18:14:40,410 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into 291fd6bb28da4c4f89c9323856d64b54(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:40,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:40,410 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=13, startTime=1731348879984; duration=0sec 2024-11-11T18:14:40,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:40,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:40,410 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:40,412 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:40,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:40,414 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:40,414 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ff85b058d48647e8b65d7faadfea3628, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/cf344ed577e04c36973edaf720d51afe, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/8b01589c83044f7fb0e39bd498c212b7] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.7 K 2024-11-11T18:14:40,414 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c4a8597cbf1a414796127d8af872cab1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c4a8597cbf1a414796127d8af872cab1 2024-11-11T18:14:40,414 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff85b058d48647e8b65d7faadfea3628, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731348876427 2024-11-11T18:14:40,415 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf344ed577e04c36973edaf720d51afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731348876784 2024-11-11T18:14:40,415 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b01589c83044f7fb0e39bd498c212b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731348877984 2024-11-11T18:14:40,419 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into c4a8597cbf1a414796127d8af872cab1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:40,419 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:40,419 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=13, startTime=1731348879984; duration=0sec 2024-11-11T18:14:40,419 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:40,419 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:40,425 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:40,426 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/d50c0c73b5744cdcb2032d2ce3145d1d is 50, key is test_row_0/C:col10/1731348879126/Put/seqid=0 2024-11-11T18:14:40,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348940425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348940426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348940426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742175_1351 (size=13119) 2024-11-11T18:14:40,516 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/49f5ce81ba514109a5a8f40bce2c3c32 2024-11-11T18:14:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/570257dfa3174e53ab1195a4f8f9a751 is 50, key is test_row_0/B:col10/1731348879144/Put/seqid=0 2024-11-11T18:14:40,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742176_1352 (size=12301) 2024-11-11T18:14:40,529 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/570257dfa3174e53ab1195a4f8f9a751 2024-11-11T18:14:40,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/eae790752e0448e3b03c844827fcbd13 is 50, key is test_row_0/C:col10/1731348879144/Put/seqid=0 2024-11-11T18:14:40,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742177_1353 (size=12301) 2024-11-11T18:14:40,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348940634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348940634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348940634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T18:14:40,839 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/d50c0c73b5744cdcb2032d2ce3145d1d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/d50c0c73b5744cdcb2032d2ce3145d1d 2024-11-11T18:14:40,844 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into d50c0c73b5744cdcb2032d2ce3145d1d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:40,844 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:40,844 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=13, startTime=1731348879984; duration=0sec 2024-11-11T18:14:40,844 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:40,844 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:40,942 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/eae790752e0448e3b03c844827fcbd13 2024-11-11T18:14:40,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348940945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348940945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/49f5ce81ba514109a5a8f40bce2c3c32 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/49f5ce81ba514109a5a8f40bce2c3c32 2024-11-11T18:14:40,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348940949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:40,953 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/49f5ce81ba514109a5a8f40bce2c3c32, entries=150, sequenceid=437, filesize=12.0 K 2024-11-11T18:14:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/570257dfa3174e53ab1195a4f8f9a751 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/570257dfa3174e53ab1195a4f8f9a751 2024-11-11T18:14:40,958 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/570257dfa3174e53ab1195a4f8f9a751, entries=150, sequenceid=437, filesize=12.0 K 2024-11-11T18:14:40,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/eae790752e0448e3b03c844827fcbd13 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eae790752e0448e3b03c844827fcbd13 2024-11-11T18:14:40,963 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eae790752e0448e3b03c844827fcbd13, entries=150, sequenceid=437, filesize=12.0 K 2024-11-11T18:14:40,963 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7ac34b3c9a264f2c66d52e98e093bf68 in 857ms, sequenceid=437, compaction requested=false 2024-11-11T18:14:40,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:40,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:40,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-11T18:14:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-11T18:14:40,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T18:14:40,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3170 sec 2024-11-11T18:14:40,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.3220 sec 2024-11-11T18:14:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:41,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T18:14:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:41,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:41,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/f64480a0874149feb3f2e13dd02d2f34 is 50, key is test_row_0/A:col10/1731348881455/Put/seqid=0 2024-11-11T18:14:41,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742178_1354 (size=12301) 2024-11-11T18:14:41,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348941475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348941476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348941478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348941584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348941584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348941585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-11T18:14:41,795 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:41,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-11T18:14:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T18:14:41,798 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:41,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:41,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:41,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348941795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348941795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348941795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:41,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:41,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/f64480a0874149feb3f2e13dd02d2f34 2024-11-11T18:14:41,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/e8e37d8ec0234feaaa87d8bf1361cbb2 is 50, key is test_row_0/B:col10/1731348881455/Put/seqid=0 2024-11-11T18:14:41,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742179_1355 (size=12301) 2024-11-11T18:14:41,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/e8e37d8ec0234feaaa87d8bf1361cbb2 2024-11-11T18:14:41,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/73554c228d394ac88270928a3ae76cb0 is 50, key is test_row_0/C:col10/1731348881455/Put/seqid=0 2024-11-11T18:14:41,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742180_1356 (size=12301) 2024-11-11T18:14:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T18:14:41,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T18:14:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:41,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T18:14:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43998 deadline: 1731348942105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:42,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44032 deadline: 1731348942105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:42,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43996 deadline: 1731348942105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:14:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T18:14:42,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T18:14:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:42,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/73554c228d394ac88270928a3ae76cb0 2024-11-11T18:14:42,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/f64480a0874149feb3f2e13dd02d2f34 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/f64480a0874149feb3f2e13dd02d2f34 2024-11-11T18:14:42,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/f64480a0874149feb3f2e13dd02d2f34, entries=150, sequenceid=459, filesize=12.0 K 2024-11-11T18:14:42,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/e8e37d8ec0234feaaa87d8bf1361cbb2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e8e37d8ec0234feaaa87d8bf1361cbb2 2024-11-11T18:14:42,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e8e37d8ec0234feaaa87d8bf1361cbb2, entries=150, sequenceid=459, filesize=12.0 K 2024-11-11T18:14:42,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/73554c228d394ac88270928a3ae76cb0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73554c228d394ac88270928a3ae76cb0 2024-11-11T18:14:42,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73554c228d394ac88270928a3ae76cb0, entries=150, sequenceid=459, filesize=12.0 K 2024-11-11T18:14:42,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7ac34b3c9a264f2c66d52e98e093bf68 in 856ms, sequenceid=459, compaction requested=true 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:42,313 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:42,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T18:14:42,314 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:42,314 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/A is initiating minor compaction (all files) 2024-11-11T18:14:42,314 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/A in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,314 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/291fd6bb28da4c4f89c9323856d64b54, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/49f5ce81ba514109a5a8f40bce2c3c32, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/f64480a0874149feb3f2e13dd02d2f34] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.8 K 2024-11-11T18:14:42,315 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 291fd6bb28da4c4f89c9323856d64b54, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731348877984 2024-11-11T18:14:42,315 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f5ce81ba514109a5a8f40bce2c3c32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731348879144 2024-11-11T18:14:42,315 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:42,315 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f64480a0874149feb3f2e13dd02d2f34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731348881455 2024-11-11T18:14:42,317 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:42,317 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/B is initiating minor compaction (all files) 2024-11-11T18:14:42,317 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/B in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,317 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c4a8597cbf1a414796127d8af872cab1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/570257dfa3174e53ab1195a4f8f9a751, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e8e37d8ec0234feaaa87d8bf1361cbb2] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.8 K 2024-11-11T18:14:42,318 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a8597cbf1a414796127d8af872cab1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731348877984 2024-11-11T18:14:42,318 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 570257dfa3174e53ab1195a4f8f9a751, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731348879144 2024-11-11T18:14:42,319 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e8e37d8ec0234feaaa87d8bf1361cbb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731348881455 2024-11-11T18:14:42,326 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#A#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:42,326 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/fa884624a0a5448a94a806658563072b is 50, key is test_row_0/A:col10/1731348881455/Put/seqid=0 2024-11-11T18:14:42,331 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#B#compaction#310 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:42,332 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c67e43770dfb49fe8441f81595725179 is 50, key is test_row_0/B:col10/1731348881455/Put/seqid=0 2024-11-11T18:14:42,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742181_1357 (size=13221) 2024-11-11T18:14:42,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742182_1358 (size=13221) 2024-11-11T18:14:42,374 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/c67e43770dfb49fe8441f81595725179 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c67e43770dfb49fe8441f81595725179 2024-11-11T18:14:42,383 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/B of 7ac34b3c9a264f2c66d52e98e093bf68 into c67e43770dfb49fe8441f81595725179(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:42,383 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:42,383 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/B, priority=13, startTime=1731348882313; duration=0sec 2024-11-11T18:14:42,383 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:42,383 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:42,383 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:42,385 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:42,385 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 7ac34b3c9a264f2c66d52e98e093bf68/C is initiating minor compaction (all files) 2024-11-11T18:14:42,385 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ac34b3c9a264f2c66d52e98e093bf68/C in TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,385 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/d50c0c73b5744cdcb2032d2ce3145d1d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eae790752e0448e3b03c844827fcbd13, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73554c228d394ac88270928a3ae76cb0] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp, totalSize=36.8 K 2024-11-11T18:14:42,385 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d50c0c73b5744cdcb2032d2ce3145d1d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731348877984 2024-11-11T18:14:42,386 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting eae790752e0448e3b03c844827fcbd13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731348879144 2024-11-11T18:14:42,386 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 73554c228d394ac88270928a3ae76cb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731348881455 2024-11-11T18:14:42,392 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ac34b3c9a264f2c66d52e98e093bf68#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:42,393 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/ec47e568d3134bc5a34bef32c5993b77 is 50, key is test_row_0/C:col10/1731348881455/Put/seqid=0 2024-11-11T18:14:42,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742183_1359 (size=13221) 2024-11-11T18:14:42,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:42,413 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:42,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:42,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/cd2de6f244db4beebc900ffc4ec60e95 is 50, key is test_row_0/A:col10/1731348881475/Put/seqid=0 2024-11-11T18:14:42,419 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/ec47e568d3134bc5a34bef32c5993b77 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ec47e568d3134bc5a34bef32c5993b77 2024-11-11T18:14:42,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742184_1360 (size=12301) 2024-11-11T18:14:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T18:14:42,427 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/C of 7ac34b3c9a264f2c66d52e98e093bf68 into ec47e568d3134bc5a34bef32c5993b77(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:42,427 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:42,427 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/C, priority=13, startTime=1731348882313; duration=0sec 2024-11-11T18:14:42,427 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:42,427 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:42,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. as already flushing 2024-11-11T18:14:42,619 INFO [Thread-1271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1271. 2024-11-11T18:14:42,619 DEBUG [Thread-1271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,619 DEBUG [Thread-1271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,620 DEBUG [Thread-1271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,620 INFO [Thread-1262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1262. 2024-11-11T18:14:42,620 DEBUG [Thread-1262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,620 DEBUG [Thread-1262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,620 DEBUG [Thread-1262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,622 INFO [Thread-1275 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1275. 2024-11-11T18:14:42,622 DEBUG [Thread-1275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,622 DEBUG [Thread-1275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,622 DEBUG [Thread-1275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,624 INFO [Thread-1273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1273. 2024-11-11T18:14:42,624 DEBUG [Thread-1273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,624 DEBUG [Thread-1273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,624 DEBUG [Thread-1273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,624 INFO [Thread-1269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1269. 2024-11-11T18:14:42,624 INFO [Thread-1264 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1264. 2024-11-11T18:14:42,624 DEBUG [Thread-1269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,624 DEBUG [Thread-1269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,624 DEBUG [Thread-1264 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,624 INFO [Thread-1260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1260. 2024-11-11T18:14:42,624 DEBUG [Thread-1264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,624 DEBUG [Thread-1260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,625 DEBUG [Thread-1260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,625 DEBUG [Thread-1269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,625 DEBUG [Thread-1264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,625 DEBUG [Thread-1260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,625 INFO [Thread-1277 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1277. 2024-11-11T18:14:42,625 DEBUG [Thread-1277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:42,625 DEBUG [Thread-1277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,625 DEBUG [Thread-1277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:42,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:42,742 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/fa884624a0a5448a94a806658563072b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fa884624a0a5448a94a806658563072b 2024-11-11T18:14:42,746 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ac34b3c9a264f2c66d52e98e093bf68/A of 7ac34b3c9a264f2c66d52e98e093bf68 into fa884624a0a5448a94a806658563072b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:42,746 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:42,746 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., storeName=7ac34b3c9a264f2c66d52e98e093bf68/A, priority=13, startTime=1731348882313; duration=0sec 2024-11-11T18:14:42,746 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:42,746 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:42,824 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/cd2de6f244db4beebc900ffc4ec60e95 2024-11-11T18:14:42,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/069dc02e8ecd48cda5125104baab31fb is 50, key is test_row_0/B:col10/1731348881475/Put/seqid=0 2024-11-11T18:14:42,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742185_1361 (size=12301) 2024-11-11T18:14:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T18:14:43,234 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/069dc02e8ecd48cda5125104baab31fb 2024-11-11T18:14:43,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/e77714e44107476cbbbbf2d2b8d6cea3 is 50, key is test_row_0/C:col10/1731348881475/Put/seqid=0 2024-11-11T18:14:43,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742186_1362 (size=12301) 2024-11-11T18:14:43,644 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/e77714e44107476cbbbbf2d2b8d6cea3 2024-11-11T18:14:43,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/cd2de6f244db4beebc900ffc4ec60e95 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/cd2de6f244db4beebc900ffc4ec60e95 2024-11-11T18:14:43,650 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/cd2de6f244db4beebc900ffc4ec60e95, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T18:14:43,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/069dc02e8ecd48cda5125104baab31fb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/069dc02e8ecd48cda5125104baab31fb 2024-11-11T18:14:43,654 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/069dc02e8ecd48cda5125104baab31fb, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T18:14:43,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/e77714e44107476cbbbbf2d2b8d6cea3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/e77714e44107476cbbbbf2d2b8d6cea3 2024-11-11T18:14:43,657 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/e77714e44107476cbbbbf2d2b8d6cea3, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T18:14:43,658 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=40.25 KB/41220 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1245ms, sequenceid=473, compaction requested=false 2024-11-11T18:14:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:43,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-11T18:14:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-11T18:14:43,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T18:14:43,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-11-11T18:14:43,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.8650 sec 2024-11-11T18:14:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T18:14:43,945 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:44,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:44,216 INFO [Thread-1258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1258. 2024-11-11T18:14:44,216 DEBUG [Thread-1258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:44,216 DEBUG [Thread-1258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:44,216 DEBUG [Thread-1258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:44,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:44,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/fdde973c156a4b95a87fe3b03047c462 is 50, key is test_row_0/A:col10/1731348882619/Put/seqid=0 2024-11-11T18:14:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742187_1363 (size=12301) 2024-11-11T18:14:44,236 INFO [Thread-1266 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1266. 2024-11-11T18:14:44,236 DEBUG [Thread-1266 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:14:44,236 DEBUG [Thread-1266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:44,236 DEBUG [Thread-1266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:44,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2833 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2897 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2755 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2834 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2922 2024-11-11T18:14:44,236 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T18:14:44,236 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:14:44,237 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:14:44,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:44,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:44,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:14:44,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T18:14:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:44,240 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348884240"}]},"ts":"1731348884240"} 2024-11-11T18:14:44,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T18:14:44,241 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T18:14:44,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:14:44,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, UNASSIGN}] 2024-11-11T18:14:44,244 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, UNASSIGN 2024-11-11T18:14:44,245 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=7ac34b3c9a264f2c66d52e98e093bf68, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:44,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, UNASSIGN because future has completed 2024-11-11T18:14:44,246 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:14:44,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:14:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:44,398 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:44,398 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:14:44,398 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 7ac34b3c9a264f2c66d52e98e093bf68, disabling compactions & flushes 2024-11-11T18:14:44,398 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:44,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/fdde973c156a4b95a87fe3b03047c462 2024-11-11T18:14:44,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/adc6d615b2fd4514bd1bb97788945ce6 is 50, key is test_row_0/B:col10/1731348882619/Put/seqid=0 2024-11-11T18:14:44,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742188_1364 (size=12301) 2024-11-11T18:14:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:45,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/adc6d615b2fd4514bd1bb97788945ce6 2024-11-11T18:14:45,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/92c0e67149c8476bb3c44c2994a18fc8 is 50, key is test_row_0/C:col10/1731348882619/Put/seqid=0 2024-11-11T18:14:45,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742189_1365 (size=12301) 2024-11-11T18:14:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:45,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/92c0e67149c8476bb3c44c2994a18fc8 2024-11-11T18:14:45,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/fdde973c156a4b95a87fe3b03047c462 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fdde973c156a4b95a87fe3b03047c462 2024-11-11T18:14:45,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fdde973c156a4b95a87fe3b03047c462, entries=150, sequenceid=485, filesize=12.0 K 2024-11-11T18:14:45,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/adc6d615b2fd4514bd1bb97788945ce6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/adc6d615b2fd4514bd1bb97788945ce6 2024-11-11T18:14:45,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/adc6d615b2fd4514bd1bb97788945ce6, entries=150, sequenceid=485, filesize=12.0 K 2024-11-11T18:14:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/92c0e67149c8476bb3c44c2994a18fc8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/92c0e67149c8476bb3c44c2994a18fc8 2024-11-11T18:14:45,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/92c0e67149c8476bb3c44c2994a18fc8, entries=150, sequenceid=485, filesize=12.0 K 2024-11-11T18:14:45,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1243ms, sequenceid=485, compaction requested=true 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ac34b3c9a264f2c66d52e98e093bf68: 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:45,459 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:45,459 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:45,459 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. because compaction request was cancelled 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:45,459 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. after waiting 0 ms 2024-11-11T18:14:45,459 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:45,459 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:A 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ac34b3c9a264f2c66d52e98e093bf68:C, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:45,459 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. because compaction request was cancelled 2024-11-11T18:14:45,459 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. because compaction request was cancelled 2024-11-11T18:14:45,459 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:C 2024-11-11T18:14:45,459 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 7ac34b3c9a264f2c66d52e98e093bf68 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-11T18:14:45,459 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ac34b3c9a264f2c66d52e98e093bf68:B 2024-11-11T18:14:45,460 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=A 2024-11-11T18:14:45,460 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:45,460 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=B 2024-11-11T18:14:45,460 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:45,460 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ac34b3c9a264f2c66d52e98e093bf68, store=C 2024-11-11T18:14:45,460 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:45,462 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/82f9f5da7a3b470f9e455f7379490bfc is 50, key is test_row_2/A:col10/1731348884235/Put/seqid=0 2024-11-11T18:14:45,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742190_1366 (size=7415) 2024-11-11T18:14:45,866 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/82f9f5da7a3b470f9e455f7379490bfc 2024-11-11T18:14:45,872 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/f00d7d3ddd604438a9503db5c1688bf9 is 50, key is test_row_2/B:col10/1731348884235/Put/seqid=0 2024-11-11T18:14:45,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742191_1367 (size=7415) 2024-11-11T18:14:46,276 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/f00d7d3ddd604438a9503db5c1688bf9 2024-11-11T18:14:46,282 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/83c08809df574c6583822882e81387b2 is 50, key is test_row_2/C:col10/1731348884235/Put/seqid=0 2024-11-11T18:14:46,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742192_1368 (size=7415) 2024-11-11T18:14:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:46,685 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/83c08809df574c6583822882e81387b2 2024-11-11T18:14:46,690 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/A/82f9f5da7a3b470f9e455f7379490bfc as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/82f9f5da7a3b470f9e455f7379490bfc 2024-11-11T18:14:46,693 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/82f9f5da7a3b470f9e455f7379490bfc, entries=50, sequenceid=489, filesize=7.2 K 2024-11-11T18:14:46,693 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/B/f00d7d3ddd604438a9503db5c1688bf9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/f00d7d3ddd604438a9503db5c1688bf9 2024-11-11T18:14:46,696 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/f00d7d3ddd604438a9503db5c1688bf9, entries=50, sequenceid=489, filesize=7.2 K 2024-11-11T18:14:46,696 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/.tmp/C/83c08809df574c6583822882e81387b2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/83c08809df574c6583822882e81387b2 2024-11-11T18:14:46,699 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/83c08809df574c6583822882e81387b2, entries=50, sequenceid=489, filesize=7.2 K 2024-11-11T18:14:46,700 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1241ms, sequenceid=489, compaction requested=true 2024-11-11T18:14:46,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/c0e9e9c731594a9b894ac5b4c57ab737, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/9edd1fb214644830a50de78077726a82, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1ff6e61edc3149248f0e93bcb54c8857, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/0786423d220c4037ae443cfba0dfa07b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/20daec4e441848569b9faac278bd4428, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d55d787c7e284a57aa3f5f23c036f2fa, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/b557477df0f643a9a419a9792dc3b966, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/225727f47aa541e09b87df0ff37190fc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7071b228c60c433ea8aee4d3e6b620b5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1907f6b45a0f442d8d1b533d4296f0e8, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/809506016b214db188bd8d53eb6e5d6b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/01803201dddf41a5b0acb7770edc56e4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3ed72a8bf9744b52a617965ca6b9380e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/a92d7afcd5ed4451a251fb7956cdb08d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5d598449c5114d419af051ca9759e76d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7c81789bf75940deabd51943814efe06, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/770e09ca3c3941188ee09661a8cc6770, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d8843d02d3d844c7a089226e416b59f0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/51e7d8563c2047dfb205d64ac6a193df, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3105113da876420abc8ae2126a19855a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/91df4725e6384d11bef93add18764e03, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/2a7c1034e25d45d48445ec6c8b4036be, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5e89b89d8b3247fba13499a0a7fcbd71, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/291fd6bb28da4c4f89c9323856d64b54, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/87bdc64f8ad1435f88214110fbe56010, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/49f5ce81ba514109a5a8f40bce2c3c32, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/f64480a0874149feb3f2e13dd02d2f34] to archive 2024-11-11T18:14:46,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:14:46,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/c0e9e9c731594a9b894ac5b4c57ab737 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/c0e9e9c731594a9b894ac5b4c57ab737 2024-11-11T18:14:46,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/9edd1fb214644830a50de78077726a82 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/9edd1fb214644830a50de78077726a82 2024-11-11T18:14:46,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1ff6e61edc3149248f0e93bcb54c8857 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1ff6e61edc3149248f0e93bcb54c8857 2024-11-11T18:14:46,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/0786423d220c4037ae443cfba0dfa07b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/0786423d220c4037ae443cfba0dfa07b 2024-11-11T18:14:46,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/20daec4e441848569b9faac278bd4428 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/20daec4e441848569b9faac278bd4428 2024-11-11T18:14:46,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d55d787c7e284a57aa3f5f23c036f2fa to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d55d787c7e284a57aa3f5f23c036f2fa 2024-11-11T18:14:46,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/b557477df0f643a9a419a9792dc3b966 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/b557477df0f643a9a419a9792dc3b966 2024-11-11T18:14:46,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/225727f47aa541e09b87df0ff37190fc to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/225727f47aa541e09b87df0ff37190fc 2024-11-11T18:14:46,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7071b228c60c433ea8aee4d3e6b620b5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7071b228c60c433ea8aee4d3e6b620b5 2024-11-11T18:14:46,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1907f6b45a0f442d8d1b533d4296f0e8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/1907f6b45a0f442d8d1b533d4296f0e8 2024-11-11T18:14:46,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/809506016b214db188bd8d53eb6e5d6b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/809506016b214db188bd8d53eb6e5d6b 2024-11-11T18:14:46,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/01803201dddf41a5b0acb7770edc56e4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/01803201dddf41a5b0acb7770edc56e4 2024-11-11T18:14:46,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3ed72a8bf9744b52a617965ca6b9380e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3ed72a8bf9744b52a617965ca6b9380e 2024-11-11T18:14:46,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/a92d7afcd5ed4451a251fb7956cdb08d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/a92d7afcd5ed4451a251fb7956cdb08d 2024-11-11T18:14:46,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5d598449c5114d419af051ca9759e76d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5d598449c5114d419af051ca9759e76d 2024-11-11T18:14:46,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7c81789bf75940deabd51943814efe06 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/7c81789bf75940deabd51943814efe06 2024-11-11T18:14:46,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/770e09ca3c3941188ee09661a8cc6770 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/770e09ca3c3941188ee09661a8cc6770 2024-11-11T18:14:46,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d8843d02d3d844c7a089226e416b59f0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/d8843d02d3d844c7a089226e416b59f0 2024-11-11T18:14:46,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/51e7d8563c2047dfb205d64ac6a193df to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/51e7d8563c2047dfb205d64ac6a193df 2024-11-11T18:14:46,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3105113da876420abc8ae2126a19855a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/3105113da876420abc8ae2126a19855a 2024-11-11T18:14:46,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/91df4725e6384d11bef93add18764e03 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/91df4725e6384d11bef93add18764e03 2024-11-11T18:14:46,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/2a7c1034e25d45d48445ec6c8b4036be to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/2a7c1034e25d45d48445ec6c8b4036be 2024-11-11T18:14:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5e89b89d8b3247fba13499a0a7fcbd71 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/5e89b89d8b3247fba13499a0a7fcbd71 2024-11-11T18:14:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/291fd6bb28da4c4f89c9323856d64b54 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/291fd6bb28da4c4f89c9323856d64b54 2024-11-11T18:14:46,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/87bdc64f8ad1435f88214110fbe56010 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/87bdc64f8ad1435f88214110fbe56010 2024-11-11T18:14:46,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/49f5ce81ba514109a5a8f40bce2c3c32 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/49f5ce81ba514109a5a8f40bce2c3c32 2024-11-11T18:14:46,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/f64480a0874149feb3f2e13dd02d2f34 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/f64480a0874149feb3f2e13dd02d2f34 2024-11-11T18:14:46,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5f2dc8e1d2a24f3aaf647d84b49830d9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/85965d824b5440d197926e45fa7f3e9a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e480089d27fd468b8abaaacb8eaf9f45, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5019139788e34a4ba98772e05a6a49ac, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/15e6b7a804394174aa7b8c9e0c684103, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4b463ad42e7d47f699c9ad2b8af31bd6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/589cd94435904357a24831ae28f9dd38, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/799db3f10eb04716b5867b1abd94046e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ae1a4c41db29418a8c40534ab3921146, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/93115ea0e85a4285bba1541413a358f7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4e413ff03fdd48568fa7154caf4c911d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/8cc2fd1a194f4a169cbd9b0b531db2c0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/0d644b0c56db4da8825d800f659ae4b9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ac88276139824c09906be8e94acc9f33, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/2917a9c385b544d096b56d39a9dbbc8e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/9d4275e6f8014228825db6e426df3aec, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/d8dccd5004f44cf3988c9bb08631b345, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/744475d7809c449fa71450695dece9f7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/7af4b02d476e4a02a9e99cf7e080901b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/212fe7c445924b76824fc25522a63ea7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/1aa538cd5bcb42b3bf0a6cab23c01c3b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5d4ee92bdec44f1eb1ce9c399c55617e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c63d43472de04b388d0e30b8e2f3ad60, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c4a8597cbf1a414796127d8af872cab1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ff945e02152e4de890ccdf5691f228b0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/570257dfa3174e53ab1195a4f8f9a751, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e8e37d8ec0234feaaa87d8bf1361cbb2] to archive 2024-11-11T18:14:46,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:14:46,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5f2dc8e1d2a24f3aaf647d84b49830d9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5f2dc8e1d2a24f3aaf647d84b49830d9 2024-11-11T18:14:46,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/85965d824b5440d197926e45fa7f3e9a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/85965d824b5440d197926e45fa7f3e9a 2024-11-11T18:14:46,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e480089d27fd468b8abaaacb8eaf9f45 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e480089d27fd468b8abaaacb8eaf9f45 2024-11-11T18:14:46,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5019139788e34a4ba98772e05a6a49ac to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5019139788e34a4ba98772e05a6a49ac 2024-11-11T18:14:46,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/15e6b7a804394174aa7b8c9e0c684103 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/15e6b7a804394174aa7b8c9e0c684103 2024-11-11T18:14:46,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4b463ad42e7d47f699c9ad2b8af31bd6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4b463ad42e7d47f699c9ad2b8af31bd6 2024-11-11T18:14:46,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/589cd94435904357a24831ae28f9dd38 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/589cd94435904357a24831ae28f9dd38 2024-11-11T18:14:46,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/799db3f10eb04716b5867b1abd94046e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/799db3f10eb04716b5867b1abd94046e 2024-11-11T18:14:46,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ae1a4c41db29418a8c40534ab3921146 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ae1a4c41db29418a8c40534ab3921146 2024-11-11T18:14:46,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/93115ea0e85a4285bba1541413a358f7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/93115ea0e85a4285bba1541413a358f7 2024-11-11T18:14:46,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4e413ff03fdd48568fa7154caf4c911d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/4e413ff03fdd48568fa7154caf4c911d 2024-11-11T18:14:46,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/8cc2fd1a194f4a169cbd9b0b531db2c0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/8cc2fd1a194f4a169cbd9b0b531db2c0 2024-11-11T18:14:46,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/0d644b0c56db4da8825d800f659ae4b9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/0d644b0c56db4da8825d800f659ae4b9 2024-11-11T18:14:46,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ac88276139824c09906be8e94acc9f33 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ac88276139824c09906be8e94acc9f33 2024-11-11T18:14:46,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/2917a9c385b544d096b56d39a9dbbc8e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/2917a9c385b544d096b56d39a9dbbc8e 2024-11-11T18:14:46,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/9d4275e6f8014228825db6e426df3aec to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/9d4275e6f8014228825db6e426df3aec 2024-11-11T18:14:46,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/d8dccd5004f44cf3988c9bb08631b345 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/d8dccd5004f44cf3988c9bb08631b345 2024-11-11T18:14:46,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/744475d7809c449fa71450695dece9f7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/744475d7809c449fa71450695dece9f7 2024-11-11T18:14:46,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/7af4b02d476e4a02a9e99cf7e080901b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/7af4b02d476e4a02a9e99cf7e080901b 2024-11-11T18:14:46,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/212fe7c445924b76824fc25522a63ea7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/212fe7c445924b76824fc25522a63ea7 2024-11-11T18:14:46,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/1aa538cd5bcb42b3bf0a6cab23c01c3b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/1aa538cd5bcb42b3bf0a6cab23c01c3b 2024-11-11T18:14:46,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5d4ee92bdec44f1eb1ce9c399c55617e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/5d4ee92bdec44f1eb1ce9c399c55617e 2024-11-11T18:14:46,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c63d43472de04b388d0e30b8e2f3ad60 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c63d43472de04b388d0e30b8e2f3ad60 2024-11-11T18:14:46,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c4a8597cbf1a414796127d8af872cab1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c4a8597cbf1a414796127d8af872cab1 2024-11-11T18:14:46,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ff945e02152e4de890ccdf5691f228b0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/ff945e02152e4de890ccdf5691f228b0 2024-11-11T18:14:46,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/570257dfa3174e53ab1195a4f8f9a751 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/570257dfa3174e53ab1195a4f8f9a751 2024-11-11T18:14:46,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e8e37d8ec0234feaaa87d8bf1361cbb2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/e8e37d8ec0234feaaa87d8bf1361cbb2 2024-11-11T18:14:46,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/29dda5da037c4375993a5256a342bd5f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/1da070f975664cef9954c32c54bf95de, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fa70f6657d2244248c4595121b32647c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/189abcc1ae124ca5ad105f8aa615a62c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/db72e7e5b7514935b5c637cf11fb791c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/589026c166564710afbce7f82d9b1053, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/bd4e940b1daa448c96f1f05e458f21ee, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/dd8f04422a44432ca8f34c72dd433323, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/be22853979364503ab92a68e25aa1743, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/77e16a9f8a1947b881c7faf042aea08b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/2490bc4ea5ce437cafc08393acc31e1a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6d4f5191334e436bbc62701b560c8959, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eb930bb446ec4175a632b990ad4b2980, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/feb709b2b7b94c2a88df952bf79d0dcc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6022fe16ebfb474bb1c6a190f38eeb12, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/c75bcb8d5fb845b98de374f20a687cc0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/3629c464e61f43478b63cf567a852b70, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73f66144961a4df6affbea25c0abb973, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fc0331e602cd4a5db570baae5c65f613, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ff85b058d48647e8b65d7faadfea3628, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/639b57eb47f1473e82d6479b2e5e7199, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/cf344ed577e04c36973edaf720d51afe, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/d50c0c73b5744cdcb2032d2ce3145d1d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/8b01589c83044f7fb0e39bd498c212b7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eae790752e0448e3b03c844827fcbd13, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73554c228d394ac88270928a3ae76cb0] to archive 2024-11-11T18:14:46,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:14:46,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/29dda5da037c4375993a5256a342bd5f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/29dda5da037c4375993a5256a342bd5f 2024-11-11T18:14:46,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/1da070f975664cef9954c32c54bf95de to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/1da070f975664cef9954c32c54bf95de 2024-11-11T18:14:46,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fa70f6657d2244248c4595121b32647c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fa70f6657d2244248c4595121b32647c 2024-11-11T18:14:46,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/189abcc1ae124ca5ad105f8aa615a62c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/189abcc1ae124ca5ad105f8aa615a62c 2024-11-11T18:14:46,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/db72e7e5b7514935b5c637cf11fb791c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/db72e7e5b7514935b5c637cf11fb791c 2024-11-11T18:14:46,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/589026c166564710afbce7f82d9b1053 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/589026c166564710afbce7f82d9b1053 2024-11-11T18:14:46,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/bd4e940b1daa448c96f1f05e458f21ee to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/bd4e940b1daa448c96f1f05e458f21ee 2024-11-11T18:14:46,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/dd8f04422a44432ca8f34c72dd433323 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/dd8f04422a44432ca8f34c72dd433323 2024-11-11T18:14:46,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/be22853979364503ab92a68e25aa1743 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/be22853979364503ab92a68e25aa1743 2024-11-11T18:14:46,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/7aa23b7ba3ab4a7da83a1b9750e9a5fd 2024-11-11T18:14:46,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/77e16a9f8a1947b881c7faf042aea08b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/77e16a9f8a1947b881c7faf042aea08b 2024-11-11T18:14:46,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/2490bc4ea5ce437cafc08393acc31e1a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/2490bc4ea5ce437cafc08393acc31e1a 2024-11-11T18:14:46,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6d4f5191334e436bbc62701b560c8959 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6d4f5191334e436bbc62701b560c8959 2024-11-11T18:14:46,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eb930bb446ec4175a632b990ad4b2980 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eb930bb446ec4175a632b990ad4b2980 2024-11-11T18:14:46,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/feb709b2b7b94c2a88df952bf79d0dcc to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/feb709b2b7b94c2a88df952bf79d0dcc 2024-11-11T18:14:46,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6022fe16ebfb474bb1c6a190f38eeb12 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/6022fe16ebfb474bb1c6a190f38eeb12 2024-11-11T18:14:46,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/c75bcb8d5fb845b98de374f20a687cc0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/c75bcb8d5fb845b98de374f20a687cc0 2024-11-11T18:14:46,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/3629c464e61f43478b63cf567a852b70 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/3629c464e61f43478b63cf567a852b70 2024-11-11T18:14:46,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73f66144961a4df6affbea25c0abb973 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73f66144961a4df6affbea25c0abb973 2024-11-11T18:14:46,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fc0331e602cd4a5db570baae5c65f613 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/fc0331e602cd4a5db570baae5c65f613 2024-11-11T18:14:46,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ff85b058d48647e8b65d7faadfea3628 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ff85b058d48647e8b65d7faadfea3628 2024-11-11T18:14:46,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/639b57eb47f1473e82d6479b2e5e7199 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/639b57eb47f1473e82d6479b2e5e7199 2024-11-11T18:14:46,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/cf344ed577e04c36973edaf720d51afe to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/cf344ed577e04c36973edaf720d51afe 2024-11-11T18:14:46,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/d50c0c73b5744cdcb2032d2ce3145d1d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/d50c0c73b5744cdcb2032d2ce3145d1d 2024-11-11T18:14:46,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/8b01589c83044f7fb0e39bd498c212b7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/8b01589c83044f7fb0e39bd498c212b7 2024-11-11T18:14:46,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eae790752e0448e3b03c844827fcbd13 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/eae790752e0448e3b03c844827fcbd13 2024-11-11T18:14:46,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73554c228d394ac88270928a3ae76cb0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/73554c228d394ac88270928a3ae76cb0 2024-11-11T18:14:46,776 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/recovered.edits/492.seqid, newMaxSeqId=492, maxSeqId=1 2024-11-11T18:14:46,777 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. 2024-11-11T18:14:46,777 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 7ac34b3c9a264f2c66d52e98e093bf68: Waiting for close lock at 1731348884398Running coprocessor pre-close hooks at 1731348884398Disabling compacts and flushes for region at 1731348884398Disabling writes for close at 1731348885459 (+1061 ms)Obtaining lock to block concurrent updates at 1731348885459Preparing flush snapshotting stores in 7ac34b3c9a264f2c66d52e98e093bf68 at 1731348885459Finished memstore snapshotting TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731348885460 (+1 ms)Flushing stores of TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68. at 1731348885460Flushing 7ac34b3c9a264f2c66d52e98e093bf68/A: creating writer at 1731348885460Flushing 7ac34b3c9a264f2c66d52e98e093bf68/A: appending metadata at 1731348885462 (+2 ms)Flushing 7ac34b3c9a264f2c66d52e98e093bf68/A: closing flushed file at 1731348885462Flushing 7ac34b3c9a264f2c66d52e98e093bf68/B: creating writer at 1731348885870 (+408 ms)Flushing 7ac34b3c9a264f2c66d52e98e093bf68/B: appending metadata at 1731348885872 (+2 ms)Flushing 7ac34b3c9a264f2c66d52e98e093bf68/B: closing flushed file at 1731348885872Flushing 7ac34b3c9a264f2c66d52e98e093bf68/C: creating writer at 1731348886279 (+407 ms)Flushing 7ac34b3c9a264f2c66d52e98e093bf68/C: appending metadata at 1731348886281 (+2 ms)Flushing 7ac34b3c9a264f2c66d52e98e093bf68/C: closing flushed file at 1731348886281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f332538: reopening flushed file at 1731348886689 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24fc0201: reopening flushed file at 1731348886693 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ad0579b: reopening flushed file at 1731348886696 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 7ac34b3c9a264f2c66d52e98e093bf68 in 1241ms, sequenceid=489, compaction requested=true at 1731348886700 (+4 ms)Writing region close event to WAL at 1731348886773 (+73 ms)Running coprocessor post-close hooks at 1731348886777 (+4 ms)Closed at 1731348886777 2024-11-11T18:14:46,779 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:46,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=7ac34b3c9a264f2c66d52e98e093bf68, regionState=CLOSED 2024-11-11T18:14:46,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:14:46,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-11T18:14:46,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 7ac34b3c9a264f2c66d52e98e093bf68, server=90f34c5bee6a,42957,1731348800579 in 2.5360 sec 2024-11-11T18:14:46,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T18:14:46,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ac34b3c9a264f2c66d52e98e093bf68, UNASSIGN in 2.5400 sec 2024-11-11T18:14:46,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T18:14:46,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5430 sec 2024-11-11T18:14:46,788 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348886788"}]},"ts":"1731348886788"} 2024-11-11T18:14:46,789 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T18:14:46,789 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T18:14:46,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5530 sec 2024-11-11T18:14:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-11T18:14:48,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:48,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T18:14:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-11T18:14:48,407 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,410 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:48,411 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/recovered.edits] 2024-11-11T18:14:48,413 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/82f9f5da7a3b470f9e455f7379490bfc to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/82f9f5da7a3b470f9e455f7379490bfc 2024-11-11T18:14:48,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/cd2de6f244db4beebc900ffc4ec60e95 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/cd2de6f244db4beebc900ffc4ec60e95 2024-11-11T18:14:48,415 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fa884624a0a5448a94a806658563072b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fa884624a0a5448a94a806658563072b 2024-11-11T18:14:48,416 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fdde973c156a4b95a87fe3b03047c462 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/A/fdde973c156a4b95a87fe3b03047c462 2024-11-11T18:14:48,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/069dc02e8ecd48cda5125104baab31fb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/069dc02e8ecd48cda5125104baab31fb 2024-11-11T18:14:48,419 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/adc6d615b2fd4514bd1bb97788945ce6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/adc6d615b2fd4514bd1bb97788945ce6 2024-11-11T18:14:48,420 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c67e43770dfb49fe8441f81595725179 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/c67e43770dfb49fe8441f81595725179 2024-11-11T18:14:48,421 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/f00d7d3ddd604438a9503db5c1688bf9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/B/f00d7d3ddd604438a9503db5c1688bf9 2024-11-11T18:14:48,422 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/83c08809df574c6583822882e81387b2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/83c08809df574c6583822882e81387b2 2024-11-11T18:14:48,423 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/92c0e67149c8476bb3c44c2994a18fc8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/92c0e67149c8476bb3c44c2994a18fc8 2024-11-11T18:14:48,424 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/e77714e44107476cbbbbf2d2b8d6cea3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/e77714e44107476cbbbbf2d2b8d6cea3 2024-11-11T18:14:48,425 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ec47e568d3134bc5a34bef32c5993b77 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/C/ec47e568d3134bc5a34bef32c5993b77 2024-11-11T18:14:48,427 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/recovered.edits/492.seqid to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68/recovered.edits/492.seqid 2024-11-11T18:14:48,427 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/7ac34b3c9a264f2c66d52e98e093bf68 2024-11-11T18:14:48,427 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T18:14:48,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,432 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T18:14:48,434 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T18:14:48,435 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,435 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T18:14:48,435 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731348888435"}]},"ts":"9223372036854775807"} 2024-11-11T18:14:48,436 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T18:14:48,436 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7ac34b3c9a264f2c66d52e98e093bf68, NAME => 'TestAcidGuarantees,,1731348861394.7ac34b3c9a264f2c66d52e98e093bf68.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T18:14:48,436 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T18:14:48,437 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731348888436"}]},"ts":"9223372036854775807"} 2024-11-11T18:14:48,438 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T18:14:48,438 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-11T18:14:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-11T18:14:48,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T18:14:48,514 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:48,523 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 252), OpenFileDescriptor=459 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 538), ProcessCount=11 (was 11), AvailableMemoryMB=8096 (was 8520) 2024-11-11T18:14:48,531 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=8096 2024-11-11T18:14:48,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:14:48,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:14:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:48,534 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T18:14:48,534 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:48,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-11T18:14:48,535 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T18:14:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T18:14:48,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742193_1369 (size=1061) 2024-11-11T18:14:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T18:14:48,809 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T18:14:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T18:14:48,941 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:14:48,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742194_1370 (size=53) 2024-11-11T18:14:49,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T18:14:49,346 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:14:49,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1d787277a166a1036d59adf968f2703c, disabling compactions & flushes 2024-11-11T18:14:49,347 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. after waiting 0 ms 2024-11-11T18:14:49,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,347 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,347 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d787277a166a1036d59adf968f2703c: Waiting for close lock at 1731348889347Disabling compacts and flushes for region at 1731348889347Disabling writes for close at 1731348889347Writing region close event to WAL at 1731348889347Closed at 1731348889347 2024-11-11T18:14:49,348 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T18:14:49,348 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731348889348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731348889348"}]},"ts":"1731348889348"} 2024-11-11T18:14:49,349 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T18:14:49,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T18:14:49,350 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348889350"}]},"ts":"1731348889350"} 2024-11-11T18:14:49,351 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T18:14:49,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, ASSIGN}] 2024-11-11T18:14:49,352 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, ASSIGN 2024-11-11T18:14:49,353 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, ASSIGN; state=OFFLINE, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=false 2024-11-11T18:14:49,504 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:49,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, ASSIGN because future has completed 2024-11-11T18:14:49,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:14:49,660 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,660 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:14:49,660 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,660 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:14:49,660 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,660 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,662 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,663 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:49,663 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d787277a166a1036d59adf968f2703c columnFamilyName A 2024-11-11T18:14:49,663 DEBUG [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:49,663 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(327): Store=1d787277a166a1036d59adf968f2703c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:49,663 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,664 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:49,664 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d787277a166a1036d59adf968f2703c columnFamilyName B 2024-11-11T18:14:49,665 DEBUG [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:49,665 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(327): Store=1d787277a166a1036d59adf968f2703c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:49,665 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,666 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:49,666 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d787277a166a1036d59adf968f2703c columnFamilyName C 2024-11-11T18:14:49,666 DEBUG [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:49,666 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(327): Store=1d787277a166a1036d59adf968f2703c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:49,666 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,666 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,667 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,667 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,668 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,668 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,668 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:14:49,669 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,671 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:14:49,671 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened 1d787277a166a1036d59adf968f2703c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71840068, jitterRate=0.0705004334449768}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:14:49,671 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:49,672 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for 1d787277a166a1036d59adf968f2703c: Running coprocessor pre-open hook at 1731348889660Writing region info on filesystem at 1731348889660Initializing all the Stores at 1731348889661 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348889661Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348889661Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348889661Cleaning up temporary data from old regions at 1731348889668 (+7 ms)Running coprocessor post-open hooks at 1731348889671 (+3 ms)Region opened successfully at 1731348889672 (+1 ms) 2024-11-11T18:14:49,672 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., pid=99, masterSystemTime=1731348889657 2024-11-11T18:14:49,674 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T18:14:49,674 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:49,674 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:49,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:14:49,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-11T18:14:49,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 in 172 msec 2024-11-11T18:14:49,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T18:14:49,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, ASSIGN in 327 msec 2024-11-11T18:14:49,680 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T18:14:49,680 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348889680"}]},"ts":"1731348889680"} 2024-11-11T18:14:49,681 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T18:14:49,682 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T18:14:49,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-11T18:14:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T18:14:50,695 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:50,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48dcb24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:50,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:50,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:50,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51052c0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:50,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c0436db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:50,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:50,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:50,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:14:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:14:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6fdbd4a9 2024-11-11T18:14:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:14:50,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:14:50,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:14:50,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:14:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T18:14:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742195_1371 (size=1097) 2024-11-11T18:14:51,115 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T18:14:51,116 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T18:14:51,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:14:51,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, REOPEN/MOVE}] 2024-11-11T18:14:51,120 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, REOPEN/MOVE 2024-11-11T18:14:51,121 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, REOPEN/MOVE because future has completed 2024-11-11T18:14:51,122 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:14:51,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:14:51,274 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,274 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:14:51,274 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing 1d787277a166a1036d59adf968f2703c, disabling compactions & flushes 2024-11-11T18:14:51,274 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,274 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,274 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. after waiting 0 ms 2024-11-11T18:14:51,274 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,278 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T18:14:51,278 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,278 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for 1d787277a166a1036d59adf968f2703c: Waiting for close lock at 1731348891274Running coprocessor pre-close hooks at 1731348891274Disabling compacts and flushes for region at 1731348891274Disabling writes for close at 1731348891274Writing region close event to WAL at 1731348891275 (+1 ms)Running coprocessor post-close hooks at 1731348891278 (+3 ms)Closed at 1731348891278 2024-11-11T18:14:51,278 WARN [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: 1d787277a166a1036d59adf968f2703c to self. 2024-11-11T18:14:51,280 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,280 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=CLOSED 2024-11-11T18:14:51,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:14:51,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-11T18:14:51,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 in 160 msec 2024-11-11T18:14:51,285 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, REOPEN/MOVE; state=CLOSED, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=true 2024-11-11T18:14:51,435 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, REOPEN/MOVE because future has completed 2024-11-11T18:14:51,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:14:51,591 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,591 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:14:51,592 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,592 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:14:51,592 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,592 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,593 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,594 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:51,594 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d787277a166a1036d59adf968f2703c columnFamilyName A 2024-11-11T18:14:51,595 DEBUG [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:51,595 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(327): Store=1d787277a166a1036d59adf968f2703c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:51,595 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,596 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:51,596 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d787277a166a1036d59adf968f2703c columnFamilyName B 2024-11-11T18:14:51,596 DEBUG [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:51,596 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(327): Store=1d787277a166a1036d59adf968f2703c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:51,596 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,597 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:14:51,597 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d787277a166a1036d59adf968f2703c columnFamilyName C 2024-11-11T18:14:51,597 DEBUG [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:51,597 INFO [StoreOpener-1d787277a166a1036d59adf968f2703c-1 {}] regionserver.HStore(327): Store=1d787277a166a1036d59adf968f2703c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:14:51,597 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,597 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,598 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,599 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,599 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,599 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,600 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:14:51,601 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,601 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened 1d787277a166a1036d59adf968f2703c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71002947, jitterRate=0.05802635848522186}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:14:51,601 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,602 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for 1d787277a166a1036d59adf968f2703c: Running coprocessor pre-open hook at 1731348891592Writing region info on filesystem at 1731348891592Initializing all the Stores at 1731348891592Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348891592Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348891593 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348891593Cleaning up temporary data from old regions at 1731348891599 (+6 ms)Running coprocessor post-open hooks at 1731348891601 (+2 ms)Region opened successfully at 1731348891602 (+1 ms) 2024-11-11T18:14:51,602 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., pid=104, masterSystemTime=1731348891589 2024-11-11T18:14:51,604 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,604 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,604 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=OPEN, openSeqNum=5, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:14:51,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-11T18:14:51,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 in 169 msec 2024-11-11T18:14:51,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T18:14:51,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, REOPEN/MOVE in 490 msec 2024-11-11T18:14:51,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-11T18:14:51,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-11T18:14:51,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-11T18:14:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-11T18:14:51,615 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:51,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5232d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,617 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1ca382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f0e8dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,620 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562103ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f2bce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6199d340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,623 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285f59e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,625 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c756a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,626 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b58dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32da4b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad5cde2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8134d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,630 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72872c9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40680bb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,633 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685e1e85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,634 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1326ee62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb4a668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6945cdef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,638 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffffd9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,640 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e52744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf0e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:14:51,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:14:51,641 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:14:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:14:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:14:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3914418a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:14:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:14:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:14:51,642 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:14:51,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e9dd07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:14:51,643 DEBUG [Thread-1675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,643 DEBUG [Thread-1671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,644 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,644 DEBUG [Thread-1686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,644 DEBUG [Thread-1677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,644 DEBUG [Thread-1684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,645 DEBUG [Thread-1682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,645 DEBUG [Thread-1669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,645 DEBUG [Thread-1688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,646 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:14:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:14:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:14:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T18:14:51,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:14:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,651 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:14:51,654 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:51,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:51,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:51,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112bf92a6a30e74673a5d87cc319740a24_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348891656/Put/seqid=0 2024-11-11T18:14:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731348891680, max timestamp=1731348891704Num uniques 859; 2024-11-11T18:14:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731348891680, max timestamp=1731348891704Num uniques 859; 2024-11-11T18:14:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731348891680, max timestamp=1731348891704Num uniques 859; 2024-11-11T18:14:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T18:14:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T18:14:51,709 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T18:14:51,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742196_1372 (size=12154) 2024-11-11T18:14:51,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348951730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348951734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348951736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348951740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348951744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:51,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348951855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348951864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348951864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348951864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348951865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:51,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:51,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:51,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348952074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348952085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348952085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348952085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348952085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:52,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:52,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:52,143 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112bf92a6a30e74673a5d87cc319740a24_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112bf92a6a30e74673a5d87cc319740a24_1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:52,144 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/50ca49e7a336408187d42d7e8774b503, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:52,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/50ca49e7a336408187d42d7e8774b503 is 175, key is test_row_0/A:col10/1731348891656/Put/seqid=0 2024-11-11T18:14:52,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742197_1373 (size=30951) 2024-11-11T18:14:52,152 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/50ca49e7a336408187d42d7e8774b503 2024-11-11T18:14:52,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/93ee7a62f2314e9190a3d0e0a5a52254 is 50, key is test_row_0/B:col10/1731348891656/Put/seqid=0 2024-11-11T18:14:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742198_1374 (size=9657) 2024-11-11T18:14:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:52,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:52,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:14:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348952395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348952395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348952395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348952404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348952404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:52,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/93ee7a62f2314e9190a3d0e0a5a52254 2024-11-11T18:14:52,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/4378535aebb44ddb8f1691786c8a83c6 is 50, key is test_row_0/C:col10/1731348891656/Put/seqid=0 2024-11-11T18:14:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742199_1375 (size=9657) 2024-11-11T18:14:52,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/4378535aebb44ddb8f1691786c8a83c6 2024-11-11T18:14:52,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/50ca49e7a336408187d42d7e8774b503 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503 2024-11-11T18:14:52,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503, entries=150, sequenceid=14, filesize=30.2 K 2024-11-11T18:14:52,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/93ee7a62f2314e9190a3d0e0a5a52254 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/93ee7a62f2314e9190a3d0e0a5a52254 2024-11-11T18:14:52,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/93ee7a62f2314e9190a3d0e0a5a52254, entries=100, sequenceid=14, filesize=9.4 K 2024-11-11T18:14:52,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/4378535aebb44ddb8f1691786c8a83c6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4378535aebb44ddb8f1691786c8a83c6 2024-11-11T18:14:52,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4378535aebb44ddb8f1691786c8a83c6, entries=100, sequenceid=14, filesize=9.4 K 2024-11-11T18:14:52,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1d787277a166a1036d59adf968f2703c in 1007ms, sequenceid=14, compaction requested=false 2024-11-11T18:14:52,667 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T18:14:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:52,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T18:14:52,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:52,737 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T18:14:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:14:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:14:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:14:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111abcccc5588594aa0b4b282309dc58903_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348891738/Put/seqid=0 2024-11-11T18:14:52,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742200_1376 (size=12154) 2024-11-11T18:14:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:52,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:52,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348952926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348952934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348952935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348952936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348952939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:52,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348953044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348953054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348953055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348953055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348953055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:53,164 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111abcccc5588594aa0b4b282309dc58903_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111abcccc5588594aa0b4b282309dc58903_1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:53,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/df460e7f1a904580a0b2445f7ddf258b, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:53,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/df460e7f1a904580a0b2445f7ddf258b is 175, key is test_row_0/A:col10/1731348891738/Put/seqid=0 2024-11-11T18:14:53,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742201_1377 (size=30955) 2024-11-11T18:14:53,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348953254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348953275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348953275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348953275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348953275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,571 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/df460e7f1a904580a0b2445f7ddf258b 2024-11-11T18:14:53,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348953564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/35f51f3a81824e91bfc10dd63dbf23e9 is 50, key is test_row_0/B:col10/1731348891738/Put/seqid=0 2024-11-11T18:14:53,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742202_1378 (size=12001) 2024-11-11T18:14:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348953585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348953585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348953585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348953594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:53,988 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/35f51f3a81824e91bfc10dd63dbf23e9 2024-11-11T18:14:53,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/187377c5b11f4000a0741a1eb159265b is 50, key is test_row_0/C:col10/1731348891738/Put/seqid=0 2024-11-11T18:14:54,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742203_1379 (size=12001) 2024-11-11T18:14:54,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348954084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:54,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348954094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:54,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348954094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348954094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348954114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:54,401 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/187377c5b11f4000a0741a1eb159265b 2024-11-11T18:14:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/df460e7f1a904580a0b2445f7ddf258b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b 2024-11-11T18:14:54,409 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b, entries=150, sequenceid=49, filesize=30.2 K 2024-11-11T18:14:54,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/35f51f3a81824e91bfc10dd63dbf23e9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/35f51f3a81824e91bfc10dd63dbf23e9 2024-11-11T18:14:54,413 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/35f51f3a81824e91bfc10dd63dbf23e9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T18:14:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/187377c5b11f4000a0741a1eb159265b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/187377c5b11f4000a0741a1eb159265b 2024-11-11T18:14:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,417 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/187377c5b11f4000a0741a1eb159265b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T18:14:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,418 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 1d787277a166a1036d59adf968f2703c in 1681ms, sequenceid=49, compaction requested=false 2024-11-11T18:14:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:54,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T18:14:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T18:14:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T18:14:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7650 sec 2024-11-11T18:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.7770 sec 2024-11-11T18:14:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:55,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:14:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:14:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:14:55,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111130b0b666d6234725960ea4e05c33dfed_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742204_1380 (size=12154) 2024-11-11T18:14:55,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:55,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348895199, max timestamp=1731348895238Num uniques 859; 2024-11-11T18:14:55,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:14:55,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348895199, max timestamp=1731348895238Num uniques 859; 2024-11-11T18:14:55,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:14:55,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348895199, max timestamp=1731348895238Num uniques 859; 2024-11-11T18:14:55,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:14:55,248 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111130b0b666d6234725960ea4e05c33dfed_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111130b0b666d6234725960ea4e05c33dfed_1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:55,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/184229c3f56a441ca585daaa235dafaf, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:55,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/184229c3f56a441ca585daaa235dafaf is 175, key is test_row_0/A:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:55,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742205_1381 (size=30951) 2024-11-11T18:14:55,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348955339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348955358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348955362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348955361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348955365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348955465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348955474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348955484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348955484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348955495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/184229c3f56a441ca585daaa235dafaf 2024-11-11T18:14:55,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348955686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348955694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/b063126d29ab4dd68167dbb66c151a47 is 50, key is test_row_0/B:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348955705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348955705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742206_1382 (size=9657) 2024-11-11T18:14:55,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/b063126d29ab4dd68167dbb66c151a47 2024-11-11T18:14:55,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348955714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:55,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/86f0921379f14a4bb5c6e13e05e16607 is 50, key is test_row_0/C:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:55,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742207_1383 (size=9657) 2024-11-11T18:14:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T18:14:55,815 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:55,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T18:14:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T18:14:55,818 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:55,819 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:55,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T18:14:55,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T18:14:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:55,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:55,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:56,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348956014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348956014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348956024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348956035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348956044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T18:14:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T18:14:56,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/86f0921379f14a4bb5c6e13e05e16607 2024-11-11T18:14:56,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/184229c3f56a441ca585daaa235dafaf as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf 2024-11-11T18:14:56,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf, entries=150, sequenceid=60, filesize=30.2 K 2024-11-11T18:14:56,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/b063126d29ab4dd68167dbb66c151a47 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/b063126d29ab4dd68167dbb66c151a47 2024-11-11T18:14:56,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/b063126d29ab4dd68167dbb66c151a47, entries=100, sequenceid=60, filesize=9.4 K 2024-11-11T18:14:56,184 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T18:14:56,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/86f0921379f14a4bb5c6e13e05e16607 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/86f0921379f14a4bb5c6e13e05e16607 2024-11-11T18:14:56,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/86f0921379f14a4bb5c6e13e05e16607, entries=100, sequenceid=60, filesize=9.4 K 2024-11-11T18:14:56,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=147.60 KB/151140 for 1d787277a166a1036d59adf968f2703c in 1050ms, sequenceid=60, compaction requested=true 2024-11-11T18:14:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:56,189 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:56,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:56,189 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:56,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:56,191 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:56,191 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/A is initiating minor compaction (all files) 2024-11-11T18:14:56,191 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/A in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,191 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=90.7 K 2024-11-11T18:14:56,191 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,191 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf] 2024-11-11T18:14:56,191 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:56,191 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/B is initiating minor compaction (all files) 2024-11-11T18:14:56,191 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/B in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,191 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/93ee7a62f2314e9190a3d0e0a5a52254, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/35f51f3a81824e91bfc10dd63dbf23e9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/b063126d29ab4dd68167dbb66c151a47] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=30.6 K 2024-11-11T18:14:56,192 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ca49e7a336408187d42d7e8774b503, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731348891656 2024-11-11T18:14:56,192 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ee7a62f2314e9190a3d0e0a5a52254, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731348891656 2024-11-11T18:14:56,192 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting df460e7f1a904580a0b2445f7ddf258b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731348891732 2024-11-11T18:14:56,192 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f51f3a81824e91bfc10dd63dbf23e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731348891732 2024-11-11T18:14:56,192 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 184229c3f56a441ca585daaa235dafaf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731348892933 2024-11-11T18:14:56,192 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b063126d29ab4dd68167dbb66c151a47, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731348892937 2024-11-11T18:14:56,205 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:56,206 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:56,206 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/73db8b6af66b4e7082e848973ea8ce0c is 50, key is test_row_0/B:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:56,229 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111195bbb30798d74412b7caea7a10847a03_1d787277a166a1036d59adf968f2703c store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:56,231 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111195bbb30798d74412b7caea7a10847a03_1d787277a166a1036d59adf968f2703c, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:56,231 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195bbb30798d74412b7caea7a10847a03_1d787277a166a1036d59adf968f2703c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:56,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742208_1384 (size=12104) 2024-11-11T18:14:56,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742209_1385 (size=4469) 2024-11-11T18:14:56,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,282 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:56,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f8b5adbdaa104c04a80c74d03b30ce39_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348895337/Put/seqid=0 2024-11-11T18:14:56,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742210_1386 (size=12154) 2024-11-11T18:14:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T18:14:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:56,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:56,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348956566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348956570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348956585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348956586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348956582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,642 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/73db8b6af66b4e7082e848973ea8ce0c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/73db8b6af66b4e7082e848973ea8ce0c 2024-11-11T18:14:56,648 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/B of 1d787277a166a1036d59adf968f2703c into 73db8b6af66b4e7082e848973ea8ce0c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:56,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:56,648 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/B, priority=13, startTime=1731348896189; duration=0sec 2024-11-11T18:14:56,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:56,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:B 2024-11-11T18:14:56,648 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:56,649 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:56,650 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/C is initiating minor compaction (all files) 2024-11-11T18:14:56,650 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/C in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:56,650 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4378535aebb44ddb8f1691786c8a83c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/187377c5b11f4000a0741a1eb159265b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/86f0921379f14a4bb5c6e13e05e16607] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=30.6 K 2024-11-11T18:14:56,652 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4378535aebb44ddb8f1691786c8a83c6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731348891656 2024-11-11T18:14:56,652 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 187377c5b11f4000a0741a1eb159265b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731348891732 2024-11-11T18:14:56,652 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 86f0921379f14a4bb5c6e13e05e16607, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731348892937 2024-11-11T18:14:56,661 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#A#compaction#331 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:56,662 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/440d8b422e0b4e7a8ade3e0a82ca606f is 175, key is test_row_0/A:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:56,668 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:56,669 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/ee4ffa4e3ab74aa7a42839a0c0a99384 is 50, key is test_row_0/C:col10/1731348892937/Put/seqid=0 2024-11-11T18:14:56,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348956694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742212_1388 (size=12104) 2024-11-11T18:14:56,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:56,722 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f8b5adbdaa104c04a80c74d03b30ce39_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8b5adbdaa104c04a80c74d03b30ce39_1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:56,722 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/ee4ffa4e3ab74aa7a42839a0c0a99384 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/ee4ffa4e3ab74aa7a42839a0c0a99384 2024-11-11T18:14:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/5193921e49ac42eca415d70194004c2f, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:56,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/5193921e49ac42eca415d70194004c2f is 175, key is test_row_0/A:col10/1731348895337/Put/seqid=0 2024-11-11T18:14:56,729 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/C of 1d787277a166a1036d59adf968f2703c into ee4ffa4e3ab74aa7a42839a0c0a99384(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:56,729 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:56,729 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/C, priority=13, startTime=1731348896190; duration=0sec 2024-11-11T18:14:56,729 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:56,729 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:C 2024-11-11T18:14:56,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348956717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348956717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348956717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742211_1387 (size=31165) 2024-11-11T18:14:56,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742213_1389 (size=30955) 2024-11-11T18:14:56,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348956925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T18:14:56,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348956945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348956945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348956945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,156 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/440d8b422e0b4e7a8ade3e0a82ca606f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/440d8b422e0b4e7a8ade3e0a82ca606f 2024-11-11T18:14:57,163 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/A of 1d787277a166a1036d59adf968f2703c into 440d8b422e0b4e7a8ade3e0a82ca606f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:57,163 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:57,163 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/A, priority=13, startTime=1731348896189; duration=0sec 2024-11-11T18:14:57,163 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:57,163 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:A 2024-11-11T18:14:57,202 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/5193921e49ac42eca415d70194004c2f 2024-11-11T18:14:57,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e2f2846f908c404d985a2080b7f64e4b is 50, key is test_row_0/B:col10/1731348895337/Put/seqid=0 2024-11-11T18:14:57,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348957254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742214_1390 (size=12001) 2024-11-11T18:14:57,270 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e2f2846f908c404d985a2080b7f64e4b 2024-11-11T18:14:57,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348957265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348957265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348957265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/253dfe3d4907429aa6bd6ca77d379bb8 is 50, key is test_row_0/C:col10/1731348895337/Put/seqid=0 2024-11-11T18:14:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742215_1391 (size=12001) 2024-11-11T18:14:57,295 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/253dfe3d4907429aa6bd6ca77d379bb8 2024-11-11T18:14:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/5193921e49ac42eca415d70194004c2f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f 2024-11-11T18:14:57,325 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f, entries=150, sequenceid=96, filesize=30.2 K 2024-11-11T18:14:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e2f2846f908c404d985a2080b7f64e4b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e2f2846f908c404d985a2080b7f64e4b 2024-11-11T18:14:57,330 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e2f2846f908c404d985a2080b7f64e4b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:14:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/253dfe3d4907429aa6bd6ca77d379bb8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/253dfe3d4907429aa6bd6ca77d379bb8 2024-11-11T18:14:57,335 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/253dfe3d4907429aa6bd6ca77d379bb8, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:14:57,338 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 1d787277a166a1036d59adf968f2703c in 1056ms, sequenceid=96, compaction requested=false 2024-11-11T18:14:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:57,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T18:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T18:14:57,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T18:14:57,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5210 sec 2024-11-11T18:14:57,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.5330 sec 2024-11-11T18:14:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:57,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:14:57,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:14:57,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:57,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:14:57,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:14:57,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:14:57,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111db7ef5439aaf4540bbcd0a44080af21c_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:57,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742216_1392 (size=14594) 2024-11-11T18:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731348897793, max timestamp=1731348897817Num uniques 859; 2024-11-11T18:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T18:14:57,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731348897793, max timestamp=1731348897817Num uniques 859; 2024-11-11T18:14:57,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T18:14:57,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731348897793, max timestamp=1731348897817Num uniques 859; 2024-11-11T18:14:57,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T18:14:57,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348957881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348957890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348957890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348957893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348957896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T18:14:57,964 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:14:57,965 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:14:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T18:14:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:14:57,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:14:57,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:14:57,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:14:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348958015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348958015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348958025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348958025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348958025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:14:58,086 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:58,090 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111db7ef5439aaf4540bbcd0a44080af21c_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111db7ef5439aaf4540bbcd0a44080af21c_1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:58,091 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/e207f1d443274a40844253bf87181d81, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:58,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/e207f1d443274a40844253bf87181d81 is 175, key is test_row_0/A:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:58,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742217_1393 (size=39549) 2024-11-11T18:14:58,109 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/e207f1d443274a40844253bf87181d81 2024-11-11T18:14:58,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/95e029c172d949748926fe59765f12a3 is 50, key is test_row_0/B:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:58,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742218_1394 (size=12001) 2024-11-11T18:14:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348958235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348958235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348958236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348958239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348958239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:14:58,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/95e029c172d949748926fe59765f12a3 2024-11-11T18:14:58,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/dfe06cf6f37e4ffa937ea37abbfe3d55 is 50, key is test_row_0/C:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:58,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742219_1395 (size=12001) 2024-11-11T18:14:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348958554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348958554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348958554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348958564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348958564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:58,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:14:58,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:58,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:14:58,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/dfe06cf6f37e4ffa937ea37abbfe3d55 2024-11-11T18:14:58,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/e207f1d443274a40844253bf87181d81 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81 2024-11-11T18:14:58,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81, entries=200, sequenceid=110, filesize=38.6 K 2024-11-11T18:14:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/95e029c172d949748926fe59765f12a3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/95e029c172d949748926fe59765f12a3 2024-11-11T18:14:58,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/95e029c172d949748926fe59765f12a3, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T18:14:58,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/dfe06cf6f37e4ffa937ea37abbfe3d55 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/dfe06cf6f37e4ffa937ea37abbfe3d55 2024-11-11T18:14:58,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/dfe06cf6f37e4ffa937ea37abbfe3d55, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T18:14:58,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1d787277a166a1036d59adf968f2703c in 1367ms, sequenceid=110, compaction requested=true 2024-11-11T18:14:58,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:14:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:14:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:14:58,973 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:58,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:58,973 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101669 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/A is initiating minor compaction (all files) 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:58,974 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/A in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/B is initiating minor compaction (all files) 2024-11-11T18:14:58,974 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/440d8b422e0b4e7a8ade3e0a82ca606f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=99.3 K 2024-11-11T18:14:58,974 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/B in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,974 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/440d8b422e0b4e7a8ade3e0a82ca606f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81] 2024-11-11T18:14:58,974 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/73db8b6af66b4e7082e848973ea8ce0c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e2f2846f908c404d985a2080b7f64e4b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/95e029c172d949748926fe59765f12a3] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=35.3 K 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 73db8b6af66b4e7082e848973ea8ce0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731348891743 2024-11-11T18:14:58,974 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440d8b422e0b4e7a8ade3e0a82ca606f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731348891743 2024-11-11T18:14:58,975 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f2846f908c404d985a2080b7f64e4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348895337 2024-11-11T18:14:58,975 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5193921e49ac42eca415d70194004c2f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348895337 2024-11-11T18:14:58,975 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e029c172d949748926fe59765f12a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731348896579 2024-11-11T18:14:58,977 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e207f1d443274a40844253bf87181d81, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731348896546 2024-11-11T18:14:58,993 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:58,995 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:58,996 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/60e7e83de1d14937a730f06be19943df is 50, key is test_row_0/B:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:59,001 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a00df70ca61143ccac7312ed1d2a2440_1d787277a166a1036d59adf968f2703c store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:59,003 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a00df70ca61143ccac7312ed1d2a2440_1d787277a166a1036d59adf968f2703c, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:59,003 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a00df70ca61143ccac7312ed1d2a2440_1d787277a166a1036d59adf968f2703c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:59,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742220_1396 (size=12207) 2024-11-11T18:14:59,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742221_1397 (size=4469) 2024-11-11T18:14:59,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T18:14:59,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:59,040 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-11T18:14:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:14:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:14:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:14:59,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:14:59,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a5c493025a944ab3ad32fcba3a9d791b_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348897889/Put/seqid=0 2024-11-11T18:14:59,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742222_1398 (size=12304) 2024-11-11T18:14:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:59,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:14:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:14:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348959094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348959095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348959096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348959102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348959103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348959214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348959214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348959214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348959214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348959225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,408 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/60e7e83de1d14937a730f06be19943df as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/60e7e83de1d14937a730f06be19943df 2024-11-11T18:14:59,408 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:59,408 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/6c15aa79c1014e88bfc897d8483025c5 is 175, key is test_row_0/A:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:59,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742223_1399 (size=31161) 2024-11-11T18:14:59,413 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/B of 1d787277a166a1036d59adf968f2703c into 60e7e83de1d14937a730f06be19943df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:59,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:59,413 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/B, priority=13, startTime=1731348898973; duration=0sec 2024-11-11T18:14:59,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:14:59,414 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:B 2024-11-11T18:14:59,414 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:14:59,415 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:14:59,415 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/C is initiating minor compaction (all files) 2024-11-11T18:14:59,415 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/C in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:14:59,415 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/ee4ffa4e3ab74aa7a42839a0c0a99384, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/253dfe3d4907429aa6bd6ca77d379bb8, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/dfe06cf6f37e4ffa937ea37abbfe3d55] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=35.3 K 2024-11-11T18:14:59,415 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ee4ffa4e3ab74aa7a42839a0c0a99384, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731348891743 2024-11-11T18:14:59,415 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 253dfe3d4907429aa6bd6ca77d379bb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348895337 2024-11-11T18:14:59,415 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/6c15aa79c1014e88bfc897d8483025c5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c15aa79c1014e88bfc897d8483025c5 2024-11-11T18:14:59,416 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe06cf6f37e4ffa937ea37abbfe3d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731348896579 2024-11-11T18:14:59,420 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/A of 1d787277a166a1036d59adf968f2703c into 6c15aa79c1014e88bfc897d8483025c5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:59,420 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:59,420 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/A, priority=13, startTime=1731348898973; duration=0sec 2024-11-11T18:14:59,420 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:59,420 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:A 2024-11-11T18:14:59,423 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:14:59,423 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/fa705eab45b347ef9a98a8f9f194a47b is 50, key is test_row_0/C:col10/1731348897598/Put/seqid=0 2024-11-11T18:14:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742224_1400 (size=12207) 2024-11-11T18:14:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348959434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348959434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348959434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348959435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:14:59,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348959444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,454 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a5c493025a944ab3ad32fcba3a9d791b_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a5c493025a944ab3ad32fcba3a9d791b_1d787277a166a1036d59adf968f2703c 2024-11-11T18:14:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/a975e15c98a0434695b05025a0469c69, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:14:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/a975e15c98a0434695b05025a0469c69 is 175, key is test_row_0/A:col10/1731348897889/Put/seqid=0 2024-11-11T18:14:59,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742225_1401 (size=31105) 2024-11-11T18:14:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348959744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348959744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348959744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348959744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:14:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348959765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:14:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:14:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:14:59,831 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/fa705eab45b347ef9a98a8f9f194a47b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/fa705eab45b347ef9a98a8f9f194a47b 2024-11-11T18:14:59,835 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/C of 1d787277a166a1036d59adf968f2703c into fa705eab45b347ef9a98a8f9f194a47b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:14:59,835 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:14:59,835 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/C, priority=13, startTime=1731348898973; duration=0sec 2024-11-11T18:14:59,835 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:14:59,835 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:C 2024-11-11T18:14:59,858 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/a975e15c98a0434695b05025a0469c69 2024-11-11T18:14:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9a567e61ae90407b93d3c00063e78d1d is 50, key is test_row_0/B:col10/1731348897889/Put/seqid=0 2024-11-11T18:14:59,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742226_1402 (size=12151) 2024-11-11T18:15:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:15:00,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348960254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:00,269 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9a567e61ae90407b93d3c00063e78d1d 2024-11-11T18:15:00,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348960265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:00,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348960265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348960265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:00,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/e2b424ebd8f1402ba044e9db824954a4 is 50, key is test_row_0/C:col10/1731348897889/Put/seqid=0 2024-11-11T18:15:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742227_1403 (size=12151) 2024-11-11T18:15:00,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348960284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:00,684 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/e2b424ebd8f1402ba044e9db824954a4 2024-11-11T18:15:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/a975e15c98a0434695b05025a0469c69 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69 2024-11-11T18:15:00,692 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69, entries=150, sequenceid=144, filesize=30.4 K 2024-11-11T18:15:00,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9a567e61ae90407b93d3c00063e78d1d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9a567e61ae90407b93d3c00063e78d1d 2024-11-11T18:15:00,696 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9a567e61ae90407b93d3c00063e78d1d, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T18:15:00,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/e2b424ebd8f1402ba044e9db824954a4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e2b424ebd8f1402ba044e9db824954a4 2024-11-11T18:15:00,705 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e2b424ebd8f1402ba044e9db824954a4, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T18:15:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 1d787277a166a1036d59adf968f2703c in 1666ms, sequenceid=144, compaction requested=false 2024-11-11T18:15:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:00,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T18:15:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T18:15:00,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T18:15:00,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7410 sec 2024-11-11T18:15:00,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.7450 sec 2024-11-11T18:15:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:01,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:15:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:01,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:01,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111365b44809312490790ef3b2c2e0e6a51_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:01,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742228_1404 (size=14794) 2024-11-11T18:15:01,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348961347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348961347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348961347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348961348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348961349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348961464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348961464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348961474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348961474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348961474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348961674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348961674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348961684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348961684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348961685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111365b44809312490790ef3b2c2e0e6a51_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111365b44809312490790ef3b2c2e0e6a51_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:01,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/bdee8c1e5ad74f8c99879c6a049093e9, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:01,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/bdee8c1e5ad74f8c99879c6a049093e9 is 175, key is test_row_0/A:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:01,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742229_1405 (size=39749) 2024-11-11T18:15:01,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348961984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348961984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348962005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348962005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348962005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,112 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/bdee8c1e5ad74f8c99879c6a049093e9 2024-11-11T18:15:02,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/7345c8e0228542fb859403f061c66430 is 50, key is test_row_0/B:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:02,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742230_1406 (size=12151) 2024-11-11T18:15:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T18:15:02,134 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:02,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T18:15:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:02,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:02,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:02,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:02,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T18:15:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T18:15:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:02,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348962504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348962504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/7345c8e0228542fb859403f061c66430 2024-11-11T18:15:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348962514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348962514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348962525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:02,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/d96a289c4f1e45fb8ce0fd48bc68b453 is 50, key is test_row_0/C:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:02,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742231_1407 (size=12151) 2024-11-11T18:15:02,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T18:15:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T18:15:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:02,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T18:15:02,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:02,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:02,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/d96a289c4f1e45fb8ce0fd48bc68b453 2024-11-11T18:15:02,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/bdee8c1e5ad74f8c99879c6a049093e9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9 2024-11-11T18:15:02,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9, entries=200, sequenceid=159, filesize=38.8 K 2024-11-11T18:15:02,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/7345c8e0228542fb859403f061c66430 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7345c8e0228542fb859403f061c66430 2024-11-11T18:15:02,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7345c8e0228542fb859403f061c66430, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T18:15:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/d96a289c4f1e45fb8ce0fd48bc68b453 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/d96a289c4f1e45fb8ce0fd48bc68b453 2024-11-11T18:15:02,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/d96a289c4f1e45fb8ce0fd48bc68b453, entries=150, sequenceid=159, filesize=11.9 K 2024-11-11T18:15:02,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1d787277a166a1036d59adf968f2703c in 1675ms, sequenceid=159, compaction requested=true 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:02,954 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:02,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:02,954 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:02,955 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:02,955 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:02,955 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/A is initiating minor compaction (all files) 2024-11-11T18:15:02,955 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/B is initiating minor compaction (all files) 2024-11-11T18:15:02,955 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/B in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,955 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/A in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,955 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/60e7e83de1d14937a730f06be19943df, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9a567e61ae90407b93d3c00063e78d1d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7345c8e0228542fb859403f061c66430] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=35.7 K 2024-11-11T18:15:02,955 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c15aa79c1014e88bfc897d8483025c5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=99.6 K 2024-11-11T18:15:02,955 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:02,955 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c15aa79c1014e88bfc897d8483025c5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9] 2024-11-11T18:15:02,955 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e7e83de1d14937a730f06be19943df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731348896579 2024-11-11T18:15:02,956 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c15aa79c1014e88bfc897d8483025c5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731348896579 2024-11-11T18:15:02,956 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a567e61ae90407b93d3c00063e78d1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731348897885 2024-11-11T18:15:02,956 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a975e15c98a0434695b05025a0469c69, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731348897885 2024-11-11T18:15:02,956 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7345c8e0228542fb859403f061c66430, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731348899100 2024-11-11T18:15:02,957 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdee8c1e5ad74f8c99879c6a049093e9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731348899095 2024-11-11T18:15:02,962 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:02,963 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:02,964 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/d884108bbad449d3885e3c0ab4f51149 is 50, key is test_row_0/B:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:02,973 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c16155c70e924ac8ae7d06ab82f8cb96_1d787277a166a1036d59adf968f2703c store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:02,975 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c16155c70e924ac8ae7d06ab82f8cb96_1d787277a166a1036d59adf968f2703c, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:02,975 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c16155c70e924ac8ae7d06ab82f8cb96_1d787277a166a1036d59adf968f2703c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:02,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742232_1408 (size=12459) 2024-11-11T18:15:02,989 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/d884108bbad449d3885e3c0ab4f51149 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d884108bbad449d3885e3c0ab4f51149 2024-11-11T18:15:02,997 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/B of 1d787277a166a1036d59adf968f2703c into d884108bbad449d3885e3c0ab4f51149(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:02,997 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:02,997 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/B, priority=13, startTime=1731348902954; duration=0sec 2024-11-11T18:15:02,997 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:02,997 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:B 2024-11-11T18:15:02,998 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:03,001 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:03,001 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/C is initiating minor compaction (all files) 2024-11-11T18:15:03,002 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/C in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:03,002 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/fa705eab45b347ef9a98a8f9f194a47b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e2b424ebd8f1402ba044e9db824954a4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/d96a289c4f1e45fb8ce0fd48bc68b453] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=35.7 K 2024-11-11T18:15:03,004 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting fa705eab45b347ef9a98a8f9f194a47b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731348896579 2024-11-11T18:15:03,004 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b424ebd8f1402ba044e9db824954a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731348897885 2024-11-11T18:15:03,005 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d96a289c4f1e45fb8ce0fd48bc68b453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731348899100 2024-11-11T18:15:03,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742233_1409 (size=4469) 2024-11-11T18:15:03,015 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#C#compaction#350 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:03,015 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/08b8256921854e6896685224e03970f9 is 50, key is test_row_0/C:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:03,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742234_1410 (size=12459) 2024-11-11T18:15:03,023 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/08b8256921854e6896685224e03970f9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/08b8256921854e6896685224e03970f9 2024-11-11T18:15:03,027 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/C of 1d787277a166a1036d59adf968f2703c into 08b8256921854e6896685224e03970f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:03,027 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:03,027 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/C, priority=13, startTime=1731348902954; duration=0sec 2024-11-11T18:15:03,027 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:03,027 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:C 2024-11-11T18:15:03,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:03,057 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:03,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113633b26dc312400fa636150ea4276b4a_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348901324/Put/seqid=0 2024-11-11T18:15:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742235_1411 (size=12304) 2024-11-11T18:15:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:03,415 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#A#compaction#349 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:03,415 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/e477825d50164247ba7e784b48e57cd2 is 175, key is test_row_0/A:col10/1731348899100/Put/seqid=0 2024-11-11T18:15:03,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742236_1412 (size=31413) 2024-11-11T18:15:03,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:03,471 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113633b26dc312400fa636150ea4276b4a_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113633b26dc312400fa636150ea4276b4a_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/39fc0f69e2334a22a2067b0122669476, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/39fc0f69e2334a22a2067b0122669476 is 175, key is test_row_0/A:col10/1731348901324/Put/seqid=0 2024-11-11T18:15:03,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742237_1413 (size=31105) 2024-11-11T18:15:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:03,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:03,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348963544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348963544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,554 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348963545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348963554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348963554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348963664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348963664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348963664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348963674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,825 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/e477825d50164247ba7e784b48e57cd2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e477825d50164247ba7e784b48e57cd2 2024-11-11T18:15:03,829 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/A of 1d787277a166a1036d59adf968f2703c into e477825d50164247ba7e784b48e57cd2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:03,829 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:03,829 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/A, priority=13, startTime=1731348902954; duration=0sec 2024-11-11T18:15:03,830 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:03,830 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:A 2024-11-11T18:15:03,876 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/39fc0f69e2334a22a2067b0122669476 2024-11-11T18:15:03,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348963874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e19a729210e8453d8926bd829017c62a is 50, key is test_row_0/B:col10/1731348901324/Put/seqid=0 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:03,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742238_1414 (size=12151) 2024-11-11T18:15:03,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348963885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348964194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348964194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348964194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348964204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,290 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e19a729210e8453d8926bd829017c62a 2024-11-11T18:15:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:04,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/e52e39923c644a7e86cff3da7f53057d is 50, key is test_row_0/C:col10/1731348901324/Put/seqid=0 2024-11-11T18:15:04,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742239_1415 (size=12151) 2024-11-11T18:15:04,704 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/e52e39923c644a7e86cff3da7f53057d 2024-11-11T18:15:04,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/39fc0f69e2334a22a2067b0122669476 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476 2024-11-11T18:15:04,711 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476, entries=150, sequenceid=181, filesize=30.4 K 2024-11-11T18:15:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e19a729210e8453d8926bd829017c62a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e19a729210e8453d8926bd829017c62a 2024-11-11T18:15:04,715 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e19a729210e8453d8926bd829017c62a, entries=150, sequenceid=181, filesize=11.9 K 2024-11-11T18:15:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/e52e39923c644a7e86cff3da7f53057d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e52e39923c644a7e86cff3da7f53057d 2024-11-11T18:15:04,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348964705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348964705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348964714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,724 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e52e39923c644a7e86cff3da7f53057d, entries=150, sequenceid=181, filesize=11.9 K 2024-11-11T18:15:04,725 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1d787277a166a1036d59adf968f2703c in 1668ms, sequenceid=181, compaction requested=false 2024-11-11T18:15:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:04,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T18:15:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T18:15:04,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T18:15:04,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5900 sec 2024-11-11T18:15:04,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.5940 sec 2024-11-11T18:15:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:04,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T18:15:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:04,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:04,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:04,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:04,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:04,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:04,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b0033834a8a84e75ba3840cbf8f00fa8_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:04,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742240_1416 (size=14794) 2024-11-11T18:15:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348964867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348964985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,148 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:05,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b0033834a8a84e75ba3840cbf8f00fa8_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b0033834a8a84e75ba3840cbf8f00fa8_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:05,152 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/d9cae686412e4ece8ca94e60da1f1ef9, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:05,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/d9cae686412e4ece8ca94e60da1f1ef9 is 175, key is test_row_0/A:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:05,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742241_1417 (size=39749) 2024-11-11T18:15:05,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/d9cae686412e4ece8ca94e60da1f1ef9 2024-11-11T18:15:05,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/1b220e330c8d4d7790fc9e1ed56b94b0 is 50, key is test_row_0/B:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:05,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742242_1418 (size=12151) 2024-11-11T18:15:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348965194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348965504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/1b220e330c8d4d7790fc9e1ed56b94b0 2024-11-11T18:15:05,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/b3184e8ef4f24ef58879a67b8bc47890 is 50, key is test_row_0/C:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:05,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348965574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:05,577 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742243_1419 (size=12151) 2024-11-11T18:15:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348965724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:05,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348965724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:05,727 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,727 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348965735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:05,740 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:05,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/b3184e8ef4f24ef58879a67b8bc47890 2024-11-11T18:15:05,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/d9cae686412e4ece8ca94e60da1f1ef9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9 2024-11-11T18:15:05,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9, entries=200, sequenceid=195, filesize=38.8 K 2024-11-11T18:15:05,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/1b220e330c8d4d7790fc9e1ed56b94b0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/1b220e330c8d4d7790fc9e1ed56b94b0 2024-11-11T18:15:05,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/1b220e330c8d4d7790fc9e1ed56b94b0, entries=150, sequenceid=195, filesize=11.9 K 2024-11-11T18:15:05,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/b3184e8ef4f24ef58879a67b8bc47890 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/b3184e8ef4f24ef58879a67b8bc47890 2024-11-11T18:15:05,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/b3184e8ef4f24ef58879a67b8bc47890, entries=150, sequenceid=195, filesize=11.9 K 2024-11-11T18:15:05,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1d787277a166a1036d59adf968f2703c in 1265ms, sequenceid=195, compaction requested=true 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:05,998 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:05,998 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:05,999 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:05,999 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:05,999 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/B is initiating minor compaction (all files) 2024-11-11T18:15:05,999 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/A is initiating minor compaction (all files) 2024-11-11T18:15:05,999 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/B in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:05,999 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/A in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:05,999 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d884108bbad449d3885e3c0ab4f51149, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e19a729210e8453d8926bd829017c62a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/1b220e330c8d4d7790fc9e1ed56b94b0] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=35.9 K 2024-11-11T18:15:05,999 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e477825d50164247ba7e784b48e57cd2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=99.9 K 2024-11-11T18:15:05,999 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:05,999 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e477825d50164247ba7e784b48e57cd2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9] 2024-11-11T18:15:05,999 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d884108bbad449d3885e3c0ab4f51149, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731348899100 2024-11-11T18:15:06,000 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e477825d50164247ba7e784b48e57cd2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731348899100 2024-11-11T18:15:06,000 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e19a729210e8453d8926bd829017c62a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731348901324 2024-11-11T18:15:06,000 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39fc0f69e2334a22a2067b0122669476, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731348901324 2024-11-11T18:15:06,000 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b220e330c8d4d7790fc9e1ed56b94b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348903531 2024-11-11T18:15:06,000 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9cae686412e4ece8ca94e60da1f1ef9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348903531 2024-11-11T18:15:06,005 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:06,006 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#B#compaction#358 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:06,007 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e9498748284d45a1be9196c9ae9f58bf is 50, key is test_row_0/B:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:06,007 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c3fddb760b164c8fa6e77d5f6b348c83_1d787277a166a1036d59adf968f2703c store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:06,009 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c3fddb760b164c8fa6e77d5f6b348c83_1d787277a166a1036d59adf968f2703c, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:06,009 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c3fddb760b164c8fa6e77d5f6b348c83_1d787277a166a1036d59adf968f2703c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:06,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742244_1420 (size=12561) 2024-11-11T18:15:06,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742245_1421 (size=4469) 2024-11-11T18:15:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:06,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:15:06,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:06,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:06,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:06,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:06,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:06,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:06,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a2222e52e6a94293b5602ffe4fae2b31_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348906026/Put/seqid=0 2024-11-11T18:15:06,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742246_1422 (size=14794) 2024-11-11T18:15:06,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348966102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:06,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348966214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T18:15:06,325 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:06,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T18:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T18:15:06,327 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:06,328 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:06,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:06,420 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#A#compaction#357 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:06,420 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/e9498748284d45a1be9196c9ae9f58bf as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e9498748284d45a1be9196c9ae9f58bf 2024-11-11T18:15:06,420 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/62e5342cf826477692dd8e621efe9dd1 is 175, key is test_row_0/A:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:06,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742247_1423 (size=31515) 2024-11-11T18:15:06,426 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/B of 1d787277a166a1036d59adf968f2703c into e9498748284d45a1be9196c9ae9f58bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:06,426 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:06,426 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/B, priority=13, startTime=1731348905998; duration=0sec 2024-11-11T18:15:06,426 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:06,426 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:B 2024-11-11T18:15:06,426 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:06,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348966424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:06,428 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:06,428 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/C is initiating minor compaction (all files) 2024-11-11T18:15:06,428 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/C in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,428 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/08b8256921854e6896685224e03970f9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e52e39923c644a7e86cff3da7f53057d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/b3184e8ef4f24ef58879a67b8bc47890] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=35.9 K 2024-11-11T18:15:06,428 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b8256921854e6896685224e03970f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731348899100 2024-11-11T18:15:06,429 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e52e39923c644a7e86cff3da7f53057d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731348901324 2024-11-11T18:15:06,429 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b3184e8ef4f24ef58879a67b8bc47890, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348903531 2024-11-11T18:15:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T18:15:06,437 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:06,438 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/679edf893c014ba2978897bc251602af is 50, key is test_row_0/C:col10/1731348903553/Put/seqid=0 2024-11-11T18:15:06,440 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:06,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742248_1424 (size=12561) 2024-11-11T18:15:06,445 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a2222e52e6a94293b5602ffe4fae2b31_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a2222e52e6a94293b5602ffe4fae2b31_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:06,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/ff6640de48354aa0b63d55023e0885ea, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:06,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/ff6640de48354aa0b63d55023e0885ea is 175, key is test_row_0/A:col10/1731348906026/Put/seqid=0 2024-11-11T18:15:06,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742249_1425 (size=39749) 2024-11-11T18:15:06,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/ff6640de48354aa0b63d55023e0885ea 2024-11-11T18:15:06,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9c5d59b8044c4c5d94e40e4abe611d18 is 50, key is test_row_0/B:col10/1731348906026/Put/seqid=0 2024-11-11T18:15:06,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742250_1426 (size=12151) 2024-11-11T18:15:06,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T18:15:06,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348966734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:06,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:06,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:06,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,829 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/62e5342cf826477692dd8e621efe9dd1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/62e5342cf826477692dd8e621efe9dd1 2024-11-11T18:15:06,833 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/A of 1d787277a166a1036d59adf968f2703c into 62e5342cf826477692dd8e621efe9dd1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:06,833 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:06,833 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/A, priority=13, startTime=1731348905998; duration=0sec 2024-11-11T18:15:06,833 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:06,833 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:A 2024-11-11T18:15:06,845 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/679edf893c014ba2978897bc251602af as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/679edf893c014ba2978897bc251602af 2024-11-11T18:15:06,849 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/C of 1d787277a166a1036d59adf968f2703c into 679edf893c014ba2978897bc251602af(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:06,849 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:06,849 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/C, priority=13, startTime=1731348905998; duration=0sec 2024-11-11T18:15:06,849 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:06,849 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:C 2024-11-11T18:15:06,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9c5d59b8044c4c5d94e40e4abe611d18 2024-11-11T18:15:06,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/c12186af99fe446ba4eeb2a480e75cc5 is 50, key is test_row_0/C:col10/1731348906026/Put/seqid=0 2024-11-11T18:15:06,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742251_1427 (size=12151) 2024-11-11T18:15:06,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:06,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:06,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T18:15:07,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:07,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:07,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:07,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:07,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348967255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:07,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/c12186af99fe446ba4eeb2a480e75cc5 2024-11-11T18:15:07,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/ff6640de48354aa0b63d55023e0885ea as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea 2024-11-11T18:15:07,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea, entries=200, sequenceid=215, filesize=38.8 K 2024-11-11T18:15:07,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9c5d59b8044c4c5d94e40e4abe611d18 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9c5d59b8044c4c5d94e40e4abe611d18 2024-11-11T18:15:07,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9c5d59b8044c4c5d94e40e4abe611d18, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T18:15:07,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/c12186af99fe446ba4eeb2a480e75cc5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c12186af99fe446ba4eeb2a480e75cc5 2024-11-11T18:15:07,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c12186af99fe446ba4eeb2a480e75cc5, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T18:15:07,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1d787277a166a1036d59adf968f2703c in 1261ms, sequenceid=215, compaction requested=false 2024-11-11T18:15:07,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:07,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T18:15:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:07,400 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:07,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111152cba2820e724a43a1b54804f52558f9_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348906089/Put/seqid=0 2024-11-11T18:15:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742252_1428 (size=12304) 2024-11-11T18:15:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:07,420 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111152cba2820e724a43a1b54804f52558f9_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111152cba2820e724a43a1b54804f52558f9_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:07,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/053409f8fa1b4e1eb47ccbe931139bf7, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:07,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/053409f8fa1b4e1eb47ccbe931139bf7 is 175, key is test_row_0/A:col10/1731348906089/Put/seqid=0 2024-11-11T18:15:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742253_1429 (size=31105) 2024-11-11T18:15:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T18:15:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:07,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:07,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348967791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348967792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348967798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:07,830 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/053409f8fa1b4e1eb47ccbe931139bf7 2024-11-11T18:15:07,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/7fbdddad89394b6ea26ac0667a186a3d is 50, key is test_row_0/B:col10/1731348906089/Put/seqid=0 2024-11-11T18:15:07,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742254_1430 (size=12151) 2024-11-11T18:15:07,866 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/7fbdddad89394b6ea26ac0667a186a3d 2024-11-11T18:15:07,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/f30d66079b424ede9a321c94c1a5cd9f is 50, key is test_row_0/C:col10/1731348906089/Put/seqid=0 2024-11-11T18:15:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348967904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:07,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742255_1431 (size=12151) 2024-11-11T18:15:07,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348967915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:07,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348967915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:07,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348968125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348968134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348968134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348968264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,274 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,325 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/f30d66079b424ede9a321c94c1a5cd9f 2024-11-11T18:15:08,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/053409f8fa1b4e1eb47ccbe931139bf7 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7 2024-11-11T18:15:08,341 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7, entries=150, sequenceid=230, filesize=30.4 K 2024-11-11T18:15:08,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/7fbdddad89394b6ea26ac0667a186a3d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7fbdddad89394b6ea26ac0667a186a3d 2024-11-11T18:15:08,346 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7fbdddad89394b6ea26ac0667a186a3d, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T18:15:08,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/f30d66079b424ede9a321c94c1a5cd9f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f30d66079b424ede9a321c94c1a5cd9f 2024-11-11T18:15:08,351 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f30d66079b424ede9a321c94c1a5cd9f, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T18:15:08,352 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1d787277a166a1036d59adf968f2703c in 953ms, sequenceid=230, compaction requested=true 2024-11-11T18:15:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T18:15:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T18:15:08,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T18:15:08,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-11T18:15:08,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.0420 sec 2024-11-11T18:15:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:08,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T18:15:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:08,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116499b4bb749d43e29e85c2b0381ed034_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:08,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742256_1432 (size=14794) 2024-11-11T18:15:08,462 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:08,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116499b4bb749d43e29e85c2b0381ed034_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116499b4bb749d43e29e85c2b0381ed034_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:08,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/63233ae51d9b4052a5638c5b7b8f7d40, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:08,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/63233ae51d9b4052a5638c5b7b8f7d40 is 175, key is test_row_0/A:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742257_1433 (size=39749) 2024-11-11T18:15:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T18:15:08,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:08,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T18:15:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:08,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:08,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:08,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348968474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348968481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348968482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348968594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348968604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348968604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348968814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348968814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:08,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348968814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:08,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/63233ae51d9b4052a5638c5b7b8f7d40 2024-11-11T18:15:08,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9e73df0ceb5742be9690ef8ba47313af is 50, key is test_row_0/B:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:08,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742258_1434 (size=12151) 2024-11-11T18:15:08,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:08,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348969134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348969134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348969134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9e73df0ceb5742be9690ef8ba47313af 2024-11-11T18:15:09,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/f734884bdffb466683dcee3c27106a00 is 50, key is test_row_0/C:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:09,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742259_1435 (size=12151) 2024-11-11T18:15:09,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:09,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:09,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:09,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59414 deadline: 1731348969624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,628 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348969644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348969644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348969644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:09,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:09,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/f734884bdffb466683dcee3c27106a00 2024-11-11T18:15:09,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:09,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/63233ae51d9b4052a5638c5b7b8f7d40 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40 2024-11-11T18:15:09,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40, entries=200, sequenceid=251, filesize=38.8 K 2024-11-11T18:15:09,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9e73df0ceb5742be9690ef8ba47313af as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9e73df0ceb5742be9690ef8ba47313af 2024-11-11T18:15:09,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9e73df0ceb5742be9690ef8ba47313af, entries=150, sequenceid=251, filesize=11.9 K 2024-11-11T18:15:09,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/f734884bdffb466683dcee3c27106a00 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f734884bdffb466683dcee3c27106a00 2024-11-11T18:15:09,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f734884bdffb466683dcee3c27106a00, entries=150, sequenceid=251, filesize=11.9 K 2024-11-11T18:15:09,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 1d787277a166a1036d59adf968f2703c in 1266ms, sequenceid=251, compaction requested=true 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:09,715 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:09,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:09,715 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:15:09,716 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:15:09,716 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:15:09,716 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/A is initiating minor compaction (all files) 2024-11-11T18:15:09,716 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/B is initiating minor compaction (all files) 2024-11-11T18:15:09,716 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/A in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,716 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/B in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,716 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/62e5342cf826477692dd8e621efe9dd1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=138.8 K 2024-11-11T18:15:09,716 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e9498748284d45a1be9196c9ae9f58bf, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9c5d59b8044c4c5d94e40e4abe611d18, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7fbdddad89394b6ea26ac0667a186a3d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9e73df0ceb5742be9690ef8ba47313af] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=47.9 K 2024-11-11T18:15:09,716 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,717 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/62e5342cf826477692dd8e621efe9dd1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40] 2024-11-11T18:15:09,717 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e9498748284d45a1be9196c9ae9f58bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348903531 2024-11-11T18:15:09,717 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e5342cf826477692dd8e621efe9dd1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348903531 2024-11-11T18:15:09,717 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5d59b8044c4c5d94e40e4abe611d18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731348904805 2024-11-11T18:15:09,718 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff6640de48354aa0b63d55023e0885ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731348904805 2024-11-11T18:15:09,718 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbdddad89394b6ea26ac0667a186a3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731348906038 2024-11-11T18:15:09,718 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 053409f8fa1b4e1eb47ccbe931139bf7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731348906038 2024-11-11T18:15:09,718 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e73df0ceb5742be9690ef8ba47313af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731348907783 2024-11-11T18:15:09,718 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63233ae51d9b4052a5638c5b7b8f7d40, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731348907783 2024-11-11T18:15:09,727 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:09,728 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9167a3f134e04d4d877abb1afa1a3bc6 is 50, key is test_row_0/B:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:09,736 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742260_1436 (size=12697) 2024-11-11T18:15:09,739 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111194b2011deb9a4451b3623dd3894317df_1d787277a166a1036d59adf968f2703c store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:09,741 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111194b2011deb9a4451b3623dd3894317df_1d787277a166a1036d59adf968f2703c, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:09,741 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111194b2011deb9a4451b3623dd3894317df_1d787277a166a1036d59adf968f2703c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:09,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742261_1437 (size=4469) 2024-11-11T18:15:09,745 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/9167a3f134e04d4d877abb1afa1a3bc6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9167a3f134e04d4d877abb1afa1a3bc6 2024-11-11T18:15:09,750 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d787277a166a1036d59adf968f2703c/B of 1d787277a166a1036d59adf968f2703c into 9167a3f134e04d4d877abb1afa1a3bc6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:09,750 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:09,750 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/B, priority=12, startTime=1731348909715; duration=0sec 2024-11-11T18:15:09,750 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:09,750 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:B 2024-11-11T18:15:09,750 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:15:09,752 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:15:09,752 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/C is initiating minor compaction (all files) 2024-11-11T18:15:09,752 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/C in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,752 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/679edf893c014ba2978897bc251602af, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c12186af99fe446ba4eeb2a480e75cc5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f30d66079b424ede9a321c94c1a5cd9f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f734884bdffb466683dcee3c27106a00] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=47.9 K 2024-11-11T18:15:09,752 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 679edf893c014ba2978897bc251602af, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348903531 2024-11-11T18:15:09,752 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c12186af99fe446ba4eeb2a480e75cc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731348904805 2024-11-11T18:15:09,753 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f30d66079b424ede9a321c94c1a5cd9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731348906038 2024-11-11T18:15:09,753 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f734884bdffb466683dcee3c27106a00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731348907783 2024-11-11T18:15:09,761 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:09,762 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/413653537a1040d5bdbc6aad5b590e8f is 50, key is test_row_0/C:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:09,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742262_1438 (size=12697) 2024-11-11T18:15:09,775 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/413653537a1040d5bdbc6aad5b590e8f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/413653537a1040d5bdbc6aad5b590e8f 2024-11-11T18:15:09,778 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d787277a166a1036d59adf968f2703c/C of 1d787277a166a1036d59adf968f2703c into 413653537a1040d5bdbc6aad5b590e8f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:09,778 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:09,778 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/C, priority=12, startTime=1731348909715; duration=0sec 2024-11-11T18:15:09,778 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:09,780 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:C 2024-11-11T18:15:09,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:09,855 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:09,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:09,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112114f4dde7e74b72a9c740930515de68_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348908473/Put/seqid=0 2024-11-11T18:15:09,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742263_1439 (size=12454) 2024-11-11T18:15:10,146 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:10,146 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/aa8477c80bb8464693356fc4b40a6d65 is 175, key is test_row_0/A:col10/1731348908448/Put/seqid=0 2024-11-11T18:15:10,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742264_1440 (size=31651) 2024-11-11T18:15:10,156 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/aa8477c80bb8464693356fc4b40a6d65 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/aa8477c80bb8464693356fc4b40a6d65 2024-11-11T18:15:10,163 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1d787277a166a1036d59adf968f2703c/A of 1d787277a166a1036d59adf968f2703c into aa8477c80bb8464693356fc4b40a6d65(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:10,163 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:10,163 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/A, priority=12, startTime=1731348909715; duration=0sec 2024-11-11T18:15:10,163 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:10,163 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:A 2024-11-11T18:15:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:10,275 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112114f4dde7e74b72a9c740930515de68_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112114f4dde7e74b72a9c740930515de68_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c1c71569521a49c088cb65e2fa6fe452, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c1c71569521a49c088cb65e2fa6fe452 is 175, key is test_row_0/A:col10/1731348908473/Put/seqid=0 2024-11-11T18:15:10,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742265_1441 (size=31255) 2024-11-11T18:15:10,282 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c1c71569521a49c088cb65e2fa6fe452 2024-11-11T18:15:10,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/6f5559fea5584176b2bcd54f407920f4 is 50, key is test_row_0/B:col10/1731348908473/Put/seqid=0 2024-11-11T18:15:10,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742266_1442 (size=12301) 2024-11-11T18:15:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:10,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. as already flushing 2024-11-11T18:15:10,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348910398, max timestamp=1731348910442Num uniques 859; 2024-11-11T18:15:10,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348910398, max timestamp=1731348910442Num uniques 859; 2024-11-11T18:15:10,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348910398, max timestamp=1731348910442Num uniques 859; 2024-11-11T18:15:10,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:15:10,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:15:10,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:15:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348970587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348970664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:10,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348970664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348970664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:10,676 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:10,677 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,677 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:10,692 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/6f5559fea5584176b2bcd54f407920f4 2024-11-11T18:15:10,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/45e55e97b2ed4197855ef8bc55a1835a is 50, key is test_row_0/C:col10/1731348908473/Put/seqid=0 2024-11-11T18:15:10,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742267_1443 (size=12301) 2024-11-11T18:15:10,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348970704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:10,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348970915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:11,103 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/45e55e97b2ed4197855ef8bc55a1835a 2024-11-11T18:15:11,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c1c71569521a49c088cb65e2fa6fe452 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452 2024-11-11T18:15:11,110 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452, entries=150, sequenceid=264, filesize=30.5 K 2024-11-11T18:15:11,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/6f5559fea5584176b2bcd54f407920f4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/6f5559fea5584176b2bcd54f407920f4 2024-11-11T18:15:11,113 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/6f5559fea5584176b2bcd54f407920f4, entries=150, sequenceid=264, filesize=12.0 K 2024-11-11T18:15:11,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/45e55e97b2ed4197855ef8bc55a1835a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/45e55e97b2ed4197855ef8bc55a1835a 2024-11-11T18:15:11,117 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/45e55e97b2ed4197855ef8bc55a1835a, entries=150, sequenceid=264, filesize=12.0 K 2024-11-11T18:15:11,118 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 1d787277a166a1036d59adf968f2703c in 1263ms, sequenceid=264, compaction requested=false 2024-11-11T18:15:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:11,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T18:15:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T18:15:11,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T18:15:11,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6420 sec 2024-11-11T18:15:11,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6460 sec 2024-11-11T18:15:11,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:11,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T18:15:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:11,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111663663370be441e19fef9b13aa5ba339_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:11,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742268_1444 (size=14994) 2024-11-11T18:15:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348971290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348971414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348971634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:11,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:11,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111663663370be441e19fef9b13aa5ba339_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111663663370be441e19fef9b13aa5ba339_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:11,650 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:11,650 INFO [Thread-1686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1686. 2024-11-11T18:15:11,650 DEBUG [Thread-1686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:11,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89 is 175, key is test_row_0/A:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:11,650 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,651 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:11,651 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-11-11T18:15:11,652 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:11,652 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,652 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:11,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742269_1445 (size=39949) 2024-11-11T18:15:11,660 INFO [Thread-1682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1682. 2024-11-11T18:15:11,660 DEBUG [Thread-1682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:11,660 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,660 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:11,663 INFO [Thread-1688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1688. 2024-11-11T18:15:11,663 DEBUG [Thread-1688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:11,663 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,663 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:11,663 INFO [Thread-1684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1684. 2024-11-11T18:15:11,663 DEBUG [Thread-1684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:11,664 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,664 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:11,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:11,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348971954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:11,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:11,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:12,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89 2024-11-11T18:15:12,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/36be20b151c048bab89b078954ec673a is 50, key is test_row_0/B:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:12,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742270_1446 (size=12301) 2024-11-11T18:15:12,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:12,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/36be20b151c048bab89b078954ec673a 2024-11-11T18:15:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59474 deadline: 1731348972464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:12,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/93f7dc73d0dd4f69aab4d8c3da918168 is 50, key is test_row_0/C:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:12,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742271_1447 (size=12301) 2024-11-11T18:15:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T18:15:12,634 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:12,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:12,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59400 deadline: 1731348972695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59496 deadline: 1731348972695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:12,695 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:12,695 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:12,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59464 deadline: 1731348972704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:12,704 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 1d787277a166a1036d59adf968f2703c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:12,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/93f7dc73d0dd4f69aab4d8c3da918168 2024-11-11T18:15:12,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89 2024-11-11T18:15:12,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89, entries=200, sequenceid=302, filesize=39.0 K 2024-11-11T18:15:12,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/36be20b151c048bab89b078954ec673a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/36be20b151c048bab89b078954ec673a 2024-11-11T18:15:12,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/36be20b151c048bab89b078954ec673a, entries=150, sequenceid=302, filesize=12.0 K 2024-11-11T18:15:12,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/93f7dc73d0dd4f69aab4d8c3da918168 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/93f7dc73d0dd4f69aab4d8c3da918168 2024-11-11T18:15:12,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/93f7dc73d0dd4f69aab4d8c3da918168, entries=150, sequenceid=302, filesize=12.0 K 2024-11-11T18:15:12,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 1d787277a166a1036d59adf968f2703c in 1652ms, sequenceid=302, compaction requested=true 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:12,886 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1d787277a166a1036d59adf968f2703c:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:12,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:12,886 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:12,887 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:12,887 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:12,887 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/B is initiating minor compaction (all files) 2024-11-11T18:15:12,887 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/A is initiating minor compaction (all files) 2024-11-11T18:15:12,887 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/A in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:12,887 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/B in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:12,887 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/aa8477c80bb8464693356fc4b40a6d65, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=100.4 K 2024-11-11T18:15:12,887 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9167a3f134e04d4d877abb1afa1a3bc6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/6f5559fea5584176b2bcd54f407920f4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/36be20b151c048bab89b078954ec673a] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=36.4 K 2024-11-11T18:15:12,887 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:12,887 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/aa8477c80bb8464693356fc4b40a6d65, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89] 2024-11-11T18:15:12,888 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9167a3f134e04d4d877abb1afa1a3bc6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731348907783 2024-11-11T18:15:12,888 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8477c80bb8464693356fc4b40a6d65, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731348907783 2024-11-11T18:15:12,888 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1c71569521a49c088cb65e2fa6fe452, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731348908464 2024-11-11T18:15:12,888 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5559fea5584176b2bcd54f407920f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731348908464 2024-11-11T18:15:12,888 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c8e6e1b1ec8497dbe9fd38a5bc58b89, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731348910495 2024-11-11T18:15:12,888 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 36be20b151c048bab89b078954ec673a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731348910532 2024-11-11T18:15:12,893 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:12,894 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:12,894 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/d27960df667c4429b80af74cf56a58e3 is 50, key is test_row_0/B:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:12,894 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fdced30264be4ef9a74468b0bbfecdc1_1d787277a166a1036d59adf968f2703c store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:12,897 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fdced30264be4ef9a74468b0bbfecdc1_1d787277a166a1036d59adf968f2703c, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:12,897 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fdced30264be4ef9a74468b0bbfecdc1_1d787277a166a1036d59adf968f2703c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:12,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742272_1448 (size=12949) 2024-11-11T18:15:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742273_1449 (size=4469) 2024-11-11T18:15:13,301 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#A#compaction#379 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:13,302 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/76b68ba3bd7b4a61b3f403e5a099cb7e is 175, key is test_row_0/A:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:13,304 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/d27960df667c4429b80af74cf56a58e3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d27960df667c4429b80af74cf56a58e3 2024-11-11T18:15:13,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742274_1450 (size=31903) 2024-11-11T18:15:13,308 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/B of 1d787277a166a1036d59adf968f2703c into d27960df667c4429b80af74cf56a58e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:13,308 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:13,308 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/B, priority=13, startTime=1731348912886; duration=0sec 2024-11-11T18:15:13,308 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:13,308 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:B 2024-11-11T18:15:13,308 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:13,309 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:13,309 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 1d787277a166a1036d59adf968f2703c/C is initiating minor compaction (all files) 2024-11-11T18:15:13,309 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1d787277a166a1036d59adf968f2703c/C in TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:13,309 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/413653537a1040d5bdbc6aad5b590e8f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/45e55e97b2ed4197855ef8bc55a1835a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/93f7dc73d0dd4f69aab4d8c3da918168] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp, totalSize=36.4 K 2024-11-11T18:15:13,309 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 413653537a1040d5bdbc6aad5b590e8f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731348907783 2024-11-11T18:15:13,309 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e55e97b2ed4197855ef8bc55a1835a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731348908464 2024-11-11T18:15:13,310 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f7dc73d0dd4f69aab4d8c3da918168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731348910532 2024-11-11T18:15:13,316 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1d787277a166a1036d59adf968f2703c#C#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:13,316 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/4d86085db0f04e01a243603682b14f43 is 50, key is test_row_0/C:col10/1731348911232/Put/seqid=0 2024-11-11T18:15:13,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742275_1451 (size=12949) 2024-11-11T18:15:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:13,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:15:13,476 INFO [Thread-1669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1669. 2024-11-11T18:15:13,476 DEBUG [Thread-1669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:13,476 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:13,476 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:13,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:13,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111597121e9c11411a963f2b6561d062fd_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348911280/Put/seqid=0 2024-11-11T18:15:13,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742276_1452 (size=12454) 2024-11-11T18:15:13,709 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/76b68ba3bd7b4a61b3f403e5a099cb7e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/76b68ba3bd7b4a61b3f403e5a099cb7e 2024-11-11T18:15:13,713 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/A of 1d787277a166a1036d59adf968f2703c into 76b68ba3bd7b4a61b3f403e5a099cb7e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:13,713 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:13,713 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/A, priority=13, startTime=1731348912886; duration=0sec 2024-11-11T18:15:13,713 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:13,713 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:A 2024-11-11T18:15:13,723 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/4d86085db0f04e01a243603682b14f43 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4d86085db0f04e01a243603682b14f43 2024-11-11T18:15:13,727 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1d787277a166a1036d59adf968f2703c/C of 1d787277a166a1036d59adf968f2703c into 4d86085db0f04e01a243603682b14f43(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:13,727 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:13,727 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., storeName=1d787277a166a1036d59adf968f2703c/C, priority=13, startTime=1731348912886; duration=0sec 2024-11-11T18:15:13,727 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:13,727 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1d787277a166a1036d59adf968f2703c:C 2024-11-11T18:15:13,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:13,888 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111597121e9c11411a963f2b6561d062fd_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111597121e9c11411a963f2b6561d062fd_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:13,889 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c9af0c1849bd43498463d87e1dfeac68, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:13,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c9af0c1849bd43498463d87e1dfeac68 is 175, key is test_row_0/A:col10/1731348911280/Put/seqid=0 2024-11-11T18:15:13,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742277_1453 (size=31255) 2024-11-11T18:15:14,293 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c9af0c1849bd43498463d87e1dfeac68 2024-11-11T18:15:14,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/879d27fca9b74945901c63a89210094d is 50, key is test_row_0/B:col10/1731348911280/Put/seqid=0 2024-11-11T18:15:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742278_1454 (size=12301) 2024-11-11T18:15:14,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/879d27fca9b74945901c63a89210094d 2024-11-11T18:15:14,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/5c7f586a565442d4b0e4ee780f428a22 is 50, key is test_row_0/C:col10/1731348911280/Put/seqid=0 2024-11-11T18:15:14,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742279_1455 (size=12301) 2024-11-11T18:15:15,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/5c7f586a565442d4b0e4ee780f428a22 2024-11-11T18:15:15,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/c9af0c1849bd43498463d87e1dfeac68 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c9af0c1849bd43498463d87e1dfeac68 2024-11-11T18:15:15,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c9af0c1849bd43498463d87e1dfeac68, entries=150, sequenceid=314, filesize=30.5 K 2024-11-11T18:15:15,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/879d27fca9b74945901c63a89210094d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/879d27fca9b74945901c63a89210094d 2024-11-11T18:15:15,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/879d27fca9b74945901c63a89210094d, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T18:15:15,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/5c7f586a565442d4b0e4ee780f428a22 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/5c7f586a565442d4b0e4ee780f428a22 2024-11-11T18:15:15,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/5c7f586a565442d4b0e4ee780f428a22, entries=150, sequenceid=314, filesize=12.0 K 2024-11-11T18:15:15,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 1d787277a166a1036d59adf968f2703c in 1648ms, sequenceid=314, compaction requested=false 2024-11-11T18:15:15,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1d787277a166a1036d59adf968f2703c: 2024-11-11T18:15:16,716 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-11-11T18:15:16,716 INFO [Thread-1677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1677. 2024-11-11T18:15:16,716 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:16,716 DEBUG [Thread-1677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:16,716 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:16,716 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:16,716 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:16,716 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:16,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:16,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:16,735 INFO [Thread-1675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1675. 2024-11-11T18:15:16,735 DEBUG [Thread-1675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:16,735 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:16,735 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:16,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:18,809 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T18:15:19,666 INFO [Thread-1671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1671. 2024-11-11T18:15:19,666 DEBUG [Thread-1671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:19,666 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:19,666 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:19,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 955 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2862 rows 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 959 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2876 rows 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 959 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2874 rows 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 963 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2886 rows 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 953 2024-11-11T18:15:19,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2856 rows 2024-11-11T18:15:19,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:15:19,667 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:15:19,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:19,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:19,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:19,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T18:15:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T18:15:19,671 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348919671"}]},"ts":"1731348919671"} 2024-11-11T18:15:19,672 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T18:15:19,672 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T18:15:19,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:15:19,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, UNASSIGN}] 2024-11-11T18:15:19,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, UNASSIGN 2024-11-11T18:15:19,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:19,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, UNASSIGN because future has completed 2024-11-11T18:15:19,676 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:15:19,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:15:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T18:15:19,828 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:19,828 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:15:19,828 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing 1d787277a166a1036d59adf968f2703c, disabling compactions & flushes 2024-11-11T18:15:19,828 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:19,828 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:19,828 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. after waiting 0 ms 2024-11-11T18:15:19,828 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:19,829 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing 1d787277a166a1036d59adf968f2703c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T18:15:19,829 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=A 2024-11-11T18:15:19,829 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:19,829 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=B 2024-11-11T18:15:19,829 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:19,829 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1d787277a166a1036d59adf968f2703c, store=C 2024-11-11T18:15:19,829 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:19,834 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cf8df8ef279b47d9a856b2749f4f2ee8_1d787277a166a1036d59adf968f2703c is 50, key is test_row_0/A:col10/1731348916715/Put/seqid=0 2024-11-11T18:15:19,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742280_1456 (size=12454) 2024-11-11T18:15:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T18:15:20,237 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:20,241 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cf8df8ef279b47d9a856b2749f4f2ee8_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf8df8ef279b47d9a856b2749f4f2ee8_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:20,242 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/3592f2bd78504688be9706ce99b02f99, store: [table=TestAcidGuarantees family=A region=1d787277a166a1036d59adf968f2703c] 2024-11-11T18:15:20,242 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/3592f2bd78504688be9706ce99b02f99 is 175, key is test_row_0/A:col10/1731348916715/Put/seqid=0 2024-11-11T18:15:20,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742281_1457 (size=31255) 2024-11-11T18:15:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T18:15:20,646 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/3592f2bd78504688be9706ce99b02f99 2024-11-11T18:15:20,652 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/ccfaa67c49ad4455bba5be79878e1b00 is 50, key is test_row_0/B:col10/1731348916715/Put/seqid=0 2024-11-11T18:15:20,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742282_1458 (size=12301) 2024-11-11T18:15:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T18:15:21,055 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/ccfaa67c49ad4455bba5be79878e1b00 2024-11-11T18:15:21,061 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/c196d0e50bf94f03a25ce67048fa8814 is 50, key is test_row_0/C:col10/1731348916715/Put/seqid=0 2024-11-11T18:15:21,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742283_1459 (size=12301) 2024-11-11T18:15:21,464 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/c196d0e50bf94f03a25ce67048fa8814 2024-11-11T18:15:21,468 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/A/3592f2bd78504688be9706ce99b02f99 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/3592f2bd78504688be9706ce99b02f99 2024-11-11T18:15:21,471 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/3592f2bd78504688be9706ce99b02f99, entries=150, sequenceid=323, filesize=30.5 K 2024-11-11T18:15:21,471 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/B/ccfaa67c49ad4455bba5be79878e1b00 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/ccfaa67c49ad4455bba5be79878e1b00 2024-11-11T18:15:21,474 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/ccfaa67c49ad4455bba5be79878e1b00, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T18:15:21,474 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/.tmp/C/c196d0e50bf94f03a25ce67048fa8814 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c196d0e50bf94f03a25ce67048fa8814 2024-11-11T18:15:21,477 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c196d0e50bf94f03a25ce67048fa8814, entries=150, sequenceid=323, filesize=12.0 K 2024-11-11T18:15:21,478 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1d787277a166a1036d59adf968f2703c in 1648ms, sequenceid=323, compaction requested=true 2024-11-11T18:15:21,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/440d8b422e0b4e7a8ade3e0a82ca606f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c15aa79c1014e88bfc897d8483025c5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e477825d50164247ba7e784b48e57cd2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/62e5342cf826477692dd8e621efe9dd1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/aa8477c80bb8464693356fc4b40a6d65, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89] to archive 2024-11-11T18:15:21,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:15:21,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/50ca49e7a336408187d42d7e8774b503 2024-11-11T18:15:21,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/df460e7f1a904580a0b2445f7ddf258b 2024-11-11T18:15:21,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/440d8b422e0b4e7a8ade3e0a82ca606f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/440d8b422e0b4e7a8ade3e0a82ca606f 2024-11-11T18:15:21,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/184229c3f56a441ca585daaa235dafaf 2024-11-11T18:15:21,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/5193921e49ac42eca415d70194004c2f 2024-11-11T18:15:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e207f1d443274a40844253bf87181d81 2024-11-11T18:15:21,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c15aa79c1014e88bfc897d8483025c5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c15aa79c1014e88bfc897d8483025c5 2024-11-11T18:15:21,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/a975e15c98a0434695b05025a0469c69 2024-11-11T18:15:21,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/bdee8c1e5ad74f8c99879c6a049093e9 2024-11-11T18:15:21,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e477825d50164247ba7e784b48e57cd2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/e477825d50164247ba7e784b48e57cd2 2024-11-11T18:15:21,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/39fc0f69e2334a22a2067b0122669476 2024-11-11T18:15:21,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/d9cae686412e4ece8ca94e60da1f1ef9 2024-11-11T18:15:21,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/62e5342cf826477692dd8e621efe9dd1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/62e5342cf826477692dd8e621efe9dd1 2024-11-11T18:15:21,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/ff6640de48354aa0b63d55023e0885ea 2024-11-11T18:15:21,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/053409f8fa1b4e1eb47ccbe931139bf7 2024-11-11T18:15:21,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/63233ae51d9b4052a5638c5b7b8f7d40 2024-11-11T18:15:21,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/aa8477c80bb8464693356fc4b40a6d65 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/aa8477c80bb8464693356fc4b40a6d65 2024-11-11T18:15:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c1c71569521a49c088cb65e2fa6fe452 2024-11-11T18:15:21,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/6c8e6e1b1ec8497dbe9fd38a5bc58b89 2024-11-11T18:15:21,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/93ee7a62f2314e9190a3d0e0a5a52254, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/35f51f3a81824e91bfc10dd63dbf23e9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/73db8b6af66b4e7082e848973ea8ce0c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/b063126d29ab4dd68167dbb66c151a47, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e2f2846f908c404d985a2080b7f64e4b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/60e7e83de1d14937a730f06be19943df, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/95e029c172d949748926fe59765f12a3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9a567e61ae90407b93d3c00063e78d1d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d884108bbad449d3885e3c0ab4f51149, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7345c8e0228542fb859403f061c66430, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e19a729210e8453d8926bd829017c62a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e9498748284d45a1be9196c9ae9f58bf, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/1b220e330c8d4d7790fc9e1ed56b94b0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9c5d59b8044c4c5d94e40e4abe611d18, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7fbdddad89394b6ea26ac0667a186a3d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9167a3f134e04d4d877abb1afa1a3bc6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9e73df0ceb5742be9690ef8ba47313af, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/6f5559fea5584176b2bcd54f407920f4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/36be20b151c048bab89b078954ec673a] to archive 2024-11-11T18:15:21,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:15:21,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/93ee7a62f2314e9190a3d0e0a5a52254 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/93ee7a62f2314e9190a3d0e0a5a52254 2024-11-11T18:15:21,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/35f51f3a81824e91bfc10dd63dbf23e9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/35f51f3a81824e91bfc10dd63dbf23e9 2024-11-11T18:15:21,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/73db8b6af66b4e7082e848973ea8ce0c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/73db8b6af66b4e7082e848973ea8ce0c 2024-11-11T18:15:21,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/b063126d29ab4dd68167dbb66c151a47 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/b063126d29ab4dd68167dbb66c151a47 2024-11-11T18:15:21,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e2f2846f908c404d985a2080b7f64e4b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e2f2846f908c404d985a2080b7f64e4b 2024-11-11T18:15:21,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/60e7e83de1d14937a730f06be19943df to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/60e7e83de1d14937a730f06be19943df 2024-11-11T18:15:21,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/95e029c172d949748926fe59765f12a3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/95e029c172d949748926fe59765f12a3 2024-11-11T18:15:21,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9a567e61ae90407b93d3c00063e78d1d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9a567e61ae90407b93d3c00063e78d1d 2024-11-11T18:15:21,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d884108bbad449d3885e3c0ab4f51149 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d884108bbad449d3885e3c0ab4f51149 2024-11-11T18:15:21,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7345c8e0228542fb859403f061c66430 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7345c8e0228542fb859403f061c66430 2024-11-11T18:15:21,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e19a729210e8453d8926bd829017c62a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e19a729210e8453d8926bd829017c62a 2024-11-11T18:15:21,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e9498748284d45a1be9196c9ae9f58bf to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/e9498748284d45a1be9196c9ae9f58bf 2024-11-11T18:15:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/1b220e330c8d4d7790fc9e1ed56b94b0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/1b220e330c8d4d7790fc9e1ed56b94b0 2024-11-11T18:15:21,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9c5d59b8044c4c5d94e40e4abe611d18 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9c5d59b8044c4c5d94e40e4abe611d18 2024-11-11T18:15:21,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7fbdddad89394b6ea26ac0667a186a3d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/7fbdddad89394b6ea26ac0667a186a3d 2024-11-11T18:15:21,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9167a3f134e04d4d877abb1afa1a3bc6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9167a3f134e04d4d877abb1afa1a3bc6 2024-11-11T18:15:21,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9e73df0ceb5742be9690ef8ba47313af to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/9e73df0ceb5742be9690ef8ba47313af 2024-11-11T18:15:21,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/6f5559fea5584176b2bcd54f407920f4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/6f5559fea5584176b2bcd54f407920f4 2024-11-11T18:15:21,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/36be20b151c048bab89b078954ec673a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/36be20b151c048bab89b078954ec673a 2024-11-11T18:15:21,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4378535aebb44ddb8f1691786c8a83c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/187377c5b11f4000a0741a1eb159265b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/ee4ffa4e3ab74aa7a42839a0c0a99384, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/86f0921379f14a4bb5c6e13e05e16607, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/253dfe3d4907429aa6bd6ca77d379bb8, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/fa705eab45b347ef9a98a8f9f194a47b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/dfe06cf6f37e4ffa937ea37abbfe3d55, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e2b424ebd8f1402ba044e9db824954a4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/08b8256921854e6896685224e03970f9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/d96a289c4f1e45fb8ce0fd48bc68b453, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e52e39923c644a7e86cff3da7f53057d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/679edf893c014ba2978897bc251602af, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/b3184e8ef4f24ef58879a67b8bc47890, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c12186af99fe446ba4eeb2a480e75cc5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f30d66079b424ede9a321c94c1a5cd9f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/413653537a1040d5bdbc6aad5b590e8f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f734884bdffb466683dcee3c27106a00, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/45e55e97b2ed4197855ef8bc55a1835a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/93f7dc73d0dd4f69aab4d8c3da918168] to archive 2024-11-11T18:15:21,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:15:21,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4378535aebb44ddb8f1691786c8a83c6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4378535aebb44ddb8f1691786c8a83c6 2024-11-11T18:15:21,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/187377c5b11f4000a0741a1eb159265b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/187377c5b11f4000a0741a1eb159265b 2024-11-11T18:15:21,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/ee4ffa4e3ab74aa7a42839a0c0a99384 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/ee4ffa4e3ab74aa7a42839a0c0a99384 2024-11-11T18:15:21,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/86f0921379f14a4bb5c6e13e05e16607 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/86f0921379f14a4bb5c6e13e05e16607 2024-11-11T18:15:21,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/253dfe3d4907429aa6bd6ca77d379bb8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/253dfe3d4907429aa6bd6ca77d379bb8 2024-11-11T18:15:21,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/fa705eab45b347ef9a98a8f9f194a47b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/fa705eab45b347ef9a98a8f9f194a47b 2024-11-11T18:15:21,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/dfe06cf6f37e4ffa937ea37abbfe3d55 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/dfe06cf6f37e4ffa937ea37abbfe3d55 2024-11-11T18:15:21,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e2b424ebd8f1402ba044e9db824954a4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e2b424ebd8f1402ba044e9db824954a4 2024-11-11T18:15:21,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/08b8256921854e6896685224e03970f9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/08b8256921854e6896685224e03970f9 2024-11-11T18:15:21,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/d96a289c4f1e45fb8ce0fd48bc68b453 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/d96a289c4f1e45fb8ce0fd48bc68b453 2024-11-11T18:15:21,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e52e39923c644a7e86cff3da7f53057d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/e52e39923c644a7e86cff3da7f53057d 2024-11-11T18:15:21,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/679edf893c014ba2978897bc251602af to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/679edf893c014ba2978897bc251602af 2024-11-11T18:15:21,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/b3184e8ef4f24ef58879a67b8bc47890 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/b3184e8ef4f24ef58879a67b8bc47890 2024-11-11T18:15:21,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c12186af99fe446ba4eeb2a480e75cc5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c12186af99fe446ba4eeb2a480e75cc5 2024-11-11T18:15:21,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f30d66079b424ede9a321c94c1a5cd9f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f30d66079b424ede9a321c94c1a5cd9f 2024-11-11T18:15:21,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/413653537a1040d5bdbc6aad5b590e8f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/413653537a1040d5bdbc6aad5b590e8f 2024-11-11T18:15:21,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f734884bdffb466683dcee3c27106a00 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/f734884bdffb466683dcee3c27106a00 2024-11-11T18:15:21,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/45e55e97b2ed4197855ef8bc55a1835a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/45e55e97b2ed4197855ef8bc55a1835a 2024-11-11T18:15:21,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/93f7dc73d0dd4f69aab4d8c3da918168 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/93f7dc73d0dd4f69aab4d8c3da918168 2024-11-11T18:15:21,529 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-11-11T18:15:21,530 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. 2024-11-11T18:15:21,530 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for 1d787277a166a1036d59adf968f2703c: Waiting for close lock at 1731348919828Running coprocessor pre-close hooks at 1731348919828Disabling compacts and flushes for region at 1731348919828Disabling writes for close at 1731348919828Obtaining lock to block concurrent updates at 1731348919829 (+1 ms)Preparing flush snapshotting stores in 1d787277a166a1036d59adf968f2703c at 1731348919829Finished memstore snapshotting TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731348919829Flushing stores of TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c. at 1731348919830 (+1 ms)Flushing 1d787277a166a1036d59adf968f2703c/A: creating writer at 1731348919830Flushing mob file 1d787277a166a1036d59adf968f2703c/A: appending metadata at 1731348919834 (+4 ms)Flushing mob file 1d787277a166a1036d59adf968f2703c/A: closing flushed file at 1731348919834Flushing 1d787277a166a1036d59adf968f2703c/A: appending metadata at 1731348920242 (+408 ms)Flushing 1d787277a166a1036d59adf968f2703c/A: closing flushed file at 1731348920242Flushing 1d787277a166a1036d59adf968f2703c/B: creating writer at 1731348920649 (+407 ms)Flushing 1d787277a166a1036d59adf968f2703c/B: appending metadata at 1731348920651 (+2 ms)Flushing 1d787277a166a1036d59adf968f2703c/B: closing flushed file at 1731348920651Flushing 1d787277a166a1036d59adf968f2703c/C: creating writer at 1731348921058 (+407 ms)Flushing 1d787277a166a1036d59adf968f2703c/C: appending metadata at 1731348921060 (+2 ms)Flushing 1d787277a166a1036d59adf968f2703c/C: closing flushed file at 1731348921060Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ce092b4: reopening flushed file at 1731348921467 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@676ab12c: reopening flushed file at 1731348921471 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fbf7cf2: reopening flushed file at 1731348921474 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1d787277a166a1036d59adf968f2703c in 1648ms, sequenceid=323, compaction requested=true at 1731348921478 (+4 ms)Writing region close event to WAL at 1731348921527 (+49 ms)Running coprocessor post-close hooks at 1731348921530 (+3 ms)Closed at 1731348921530 2024-11-11T18:15:21,532 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed 1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,532 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=1d787277a166a1036d59adf968f2703c, regionState=CLOSED 2024-11-11T18:15:21,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:15:21,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T18:15:21,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure 1d787277a166a1036d59adf968f2703c, server=90f34c5bee6a,42957,1731348800579 in 1.8580 sec 2024-11-11T18:15:21,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-11T18:15:21,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1d787277a166a1036d59adf968f2703c, UNASSIGN in 1.8610 sec 2024-11-11T18:15:21,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T18:15:21,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8640 sec 2024-11-11T18:15:21,539 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348921539"}]},"ts":"1731348921539"} 2024-11-11T18:15:21,540 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T18:15:21,540 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T18:15:21,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8730 sec 2024-11-11T18:15:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T18:15:21,815 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:21,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T18:15:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,817 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T18:15:21,818 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,819 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,821 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/recovered.edits] 2024-11-11T18:15:21,823 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/3592f2bd78504688be9706ce99b02f99 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/3592f2bd78504688be9706ce99b02f99 2024-11-11T18:15:21,824 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/76b68ba3bd7b4a61b3f403e5a099cb7e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/76b68ba3bd7b4a61b3f403e5a099cb7e 2024-11-11T18:15:21,825 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c9af0c1849bd43498463d87e1dfeac68 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/A/c9af0c1849bd43498463d87e1dfeac68 2024-11-11T18:15:21,826 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/879d27fca9b74945901c63a89210094d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/879d27fca9b74945901c63a89210094d 2024-11-11T18:15:21,827 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/ccfaa67c49ad4455bba5be79878e1b00 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/ccfaa67c49ad4455bba5be79878e1b00 2024-11-11T18:15:21,828 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d27960df667c4429b80af74cf56a58e3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/B/d27960df667c4429b80af74cf56a58e3 2024-11-11T18:15:21,830 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4d86085db0f04e01a243603682b14f43 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/4d86085db0f04e01a243603682b14f43 2024-11-11T18:15:21,831 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/5c7f586a565442d4b0e4ee780f428a22 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/5c7f586a565442d4b0e4ee780f428a22 2024-11-11T18:15:21,832 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c196d0e50bf94f03a25ce67048fa8814 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/C/c196d0e50bf94f03a25ce67048fa8814 2024-11-11T18:15:21,834 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/recovered.edits/326.seqid to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c/recovered.edits/326.seqid 2024-11-11T18:15:21,834 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,834 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T18:15:21,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T18:15:21,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T18:15:21,837 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111597121e9c11411a963f2b6561d062fd_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111597121e9c11411a963f2b6561d062fd_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,838 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112114f4dde7e74b72a9c740930515de68_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112114f4dde7e74b72a9c740930515de68_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,839 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112bf92a6a30e74673a5d87cc319740a24_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112bf92a6a30e74673a5d87cc319740a24_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,840 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111130b0b666d6234725960ea4e05c33dfed_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111130b0b666d6234725960ea4e05c33dfed_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,841 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113633b26dc312400fa636150ea4276b4a_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113633b26dc312400fa636150ea4276b4a_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,842 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111365b44809312490790ef3b2c2e0e6a51_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111365b44809312490790ef3b2c2e0e6a51_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,843 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111152cba2820e724a43a1b54804f52558f9_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111152cba2820e724a43a1b54804f52558f9_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,844 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116499b4bb749d43e29e85c2b0381ed034_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116499b4bb749d43e29e85c2b0381ed034_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,845 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111663663370be441e19fef9b13aa5ba339_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111663663370be441e19fef9b13aa5ba339_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,845 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a2222e52e6a94293b5602ffe4fae2b31_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a2222e52e6a94293b5602ffe4fae2b31_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,846 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a5c493025a944ab3ad32fcba3a9d791b_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a5c493025a944ab3ad32fcba3a9d791b_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,847 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111abcccc5588594aa0b4b282309dc58903_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111abcccc5588594aa0b4b282309dc58903_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,848 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b0033834a8a84e75ba3840cbf8f00fa8_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b0033834a8a84e75ba3840cbf8f00fa8_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,849 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf8df8ef279b47d9a856b2749f4f2ee8_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cf8df8ef279b47d9a856b2749f4f2ee8_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,850 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111db7ef5439aaf4540bbcd0a44080af21c_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111db7ef5439aaf4540bbcd0a44080af21c_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,850 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8b5adbdaa104c04a80c74d03b30ce39_1d787277a166a1036d59adf968f2703c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f8b5adbdaa104c04a80c74d03b30ce39_1d787277a166a1036d59adf968f2703c 2024-11-11T18:15:21,851 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T18:15:21,853 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,855 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T18:15:21,857 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T18:15:21,858 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,858 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T18:15:21,858 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731348921858"}]},"ts":"9223372036854775807"} 2024-11-11T18:15:21,860 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T18:15:21,860 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1d787277a166a1036d59adf968f2703c, NAME => 'TestAcidGuarantees,,1731348888532.1d787277a166a1036d59adf968f2703c.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T18:15:21,860 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T18:15:21,860 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731348921860"}]},"ts":"9223372036854775807"} 2024-11-11T18:15:21,861 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T18:15:21,862 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-11T18:15:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T18:15:21,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T18:15:21,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:21,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 252), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=7587 (was 8096) 2024-11-11T18:15:21,943 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=7587 2024-11-11T18:15:21,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:15:21,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:15:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:21,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T18:15:21,946 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:21,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-11T18:15:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T18:15:21,947 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T18:15:21,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742284_1460 (size=1061) 2024-11-11T18:15:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T18:15:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T18:15:22,353 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:15:22,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742285_1461 (size=53) 2024-11-11T18:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T18:15:22,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:15:22,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 61e498c0de1e69a7e5cebf09f9286b5a, disabling compactions & flushes 2024-11-11T18:15:22,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:22,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:22,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. after waiting 0 ms 2024-11-11T18:15:22,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:22,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:22,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 61e498c0de1e69a7e5cebf09f9286b5a: Waiting for close lock at 1731348922758Disabling compacts and flushes for region at 1731348922758Disabling writes for close at 1731348922758Writing region close event to WAL at 1731348922758Closed at 1731348922758 2024-11-11T18:15:22,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T18:15:22,760 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731348922759"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731348922759"}]},"ts":"1731348922759"} 2024-11-11T18:15:22,761 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T18:15:22,761 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T18:15:22,762 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348922761"}]},"ts":"1731348922761"} 2024-11-11T18:15:22,762 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T18:15:22,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, ASSIGN}] 2024-11-11T18:15:22,764 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, ASSIGN 2024-11-11T18:15:22,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, ASSIGN; state=OFFLINE, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=false 2024-11-11T18:15:22,915 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=61e498c0de1e69a7e5cebf09f9286b5a, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:22,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, ASSIGN because future has completed 2024-11-11T18:15:22,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:15:23,071 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:23,071 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:15:23,072 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,072 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:15:23,072 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,072 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,073 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,074 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:23,074 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61e498c0de1e69a7e5cebf09f9286b5a columnFamilyName A 2024-11-11T18:15:23,074 DEBUG [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:23,075 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.HStore(327): Store=61e498c0de1e69a7e5cebf09f9286b5a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:23,075 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,075 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:23,076 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61e498c0de1e69a7e5cebf09f9286b5a columnFamilyName B 2024-11-11T18:15:23,076 DEBUG [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:23,076 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.HStore(327): Store=61e498c0de1e69a7e5cebf09f9286b5a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:23,076 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,077 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:23,077 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61e498c0de1e69a7e5cebf09f9286b5a columnFamilyName C 2024-11-11T18:15:23,077 DEBUG [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:23,077 INFO [StoreOpener-61e498c0de1e69a7e5cebf09f9286b5a-1 {}] regionserver.HStore(327): Store=61e498c0de1e69a7e5cebf09f9286b5a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:23,077 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:23,077 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,078 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,078 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,079 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,079 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,079 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:15:23,080 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,081 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:15:23,081 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 61e498c0de1e69a7e5cebf09f9286b5a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67651407, jitterRate=0.008084520697593689}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:15:23,082 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:23,082 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 61e498c0de1e69a7e5cebf09f9286b5a: Running coprocessor pre-open hook at 1731348923072Writing region info on filesystem at 1731348923072Initializing all the Stores at 1731348923072Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348923073 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348923073Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348923073Cleaning up temporary data from old regions at 1731348923079 (+6 ms)Running coprocessor post-open hooks at 1731348923082 (+3 ms)Region opened successfully at 1731348923082 2024-11-11T18:15:23,083 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., pid=124, masterSystemTime=1731348923068 2024-11-11T18:15:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T18:15:23,084 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:23,084 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:23,085 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=61e498c0de1e69a7e5cebf09f9286b5a, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:23,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:15:23,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T18:15:23,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 in 170 msec 2024-11-11T18:15:23,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-11T18:15:23,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, ASSIGN in 325 msec 2024-11-11T18:15:23,090 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T18:15:23,090 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348923090"}]},"ts":"1731348923090"} 2024-11-11T18:15:23,091 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T18:15:23,091 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T18:15:23,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-11T18:15:23,142 WARN [regionserver/90f34c5bee6a:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-11T18:15:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T18:15:24,095 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:24,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@845334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65353097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca990c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,102 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:15:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d3af1c9 2024-11-11T18:15:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:15:24,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:15:24,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d3d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,105 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228c3ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,106 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2adf38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3734923a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,108 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e7c77c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442006f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,111 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a96a26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb6b9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,114 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c80e575, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,116 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4852d2e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,118 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330d6b9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,119 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b71374e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,121 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd4b7bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,122 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6763777a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f86346d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,123 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42826064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,125 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f01298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,127 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f97dad7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,128 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3a2c45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,129 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a005683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,131 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3d5ae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,132 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:24,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:24,132 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:24,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c7a0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:24,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:24,133 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28088861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:24,134 DEBUG [Thread-2077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,134 DEBUG [Thread-2062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,135 DEBUG [Thread-2073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,135 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,135 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,136 DEBUG [Thread-2064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,136 DEBUG [Thread-2075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,137 DEBUG [Thread-2064 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,137 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,137 DEBUG [Thread-2064 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,140 DEBUG [Thread-2066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,142 DEBUG [Thread-2060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,149 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:24,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:24,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:24,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2] 2024-11-11T18:15:24,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:15:24,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:24,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:24,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:24,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T18:15:24,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T18:15:24,158 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:24,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:24,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/8cd22704734a49b1929b3962c84cc680 is 50, key is test_row_0/A:col10/1731348924148/Put/seqid=0 2024-11-11T18:15:24,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742286_1462 (size=14341) 2024-11-11T18:15:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348984185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348984186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348984189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348924182, max timestamp=1731348924199Num uniques 859; 2024-11-11T18:15:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:15:24,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348924182, max timestamp=1731348924199Num uniques 859; 2024-11-11T18:15:24,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:15:24,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348924182, max timestamp=1731348924199Num uniques 859; 2024-11-11T18:15:24,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T18:15:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348984279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348984295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T18:15:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348984314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348984315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348984317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348984404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348984414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T18:15:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T18:15:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348984535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348984535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348984535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/8cd22704734a49b1929b3962c84cc680 2024-11-11T18:15:24,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T18:15:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/10b04d99ed844118a740d1a751db15cc is 50, key is test_row_0/B:col10/1731348924148/Put/seqid=0 2024-11-11T18:15:24,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742287_1463 (size=12001) 2024-11-11T18:15:24,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348984624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348984635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T18:15:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T18:15:24,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348984854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348984854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348984854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T18:15:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:24,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348984944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348984944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/10b04d99ed844118a740d1a751db15cc 2024-11-11T18:15:25,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/44a0c5f35ad04f32b9daa4aa6e61a7e5 is 50, key is test_row_0/C:col10/1731348924148/Put/seqid=0 2024-11-11T18:15:25,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742288_1464 (size=12001) 2024-11-11T18:15:25,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/44a0c5f35ad04f32b9daa4aa6e61a7e5 2024-11-11T18:15:25,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/8cd22704734a49b1929b3962c84cc680 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8cd22704734a49b1929b3962c84cc680 2024-11-11T18:15:25,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8cd22704734a49b1929b3962c84cc680, entries=200, sequenceid=12, filesize=14.0 K 2024-11-11T18:15:25,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/10b04d99ed844118a740d1a751db15cc as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/10b04d99ed844118a740d1a751db15cc 2024-11-11T18:15:25,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/10b04d99ed844118a740d1a751db15cc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T18:15:25,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/44a0c5f35ad04f32b9daa4aa6e61a7e5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44a0c5f35ad04f32b9daa4aa6e61a7e5 2024-11-11T18:15:25,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44a0c5f35ad04f32b9daa4aa6e61a7e5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T18:15:25,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 61e498c0de1e69a7e5cebf09f9286b5a in 911ms, sequenceid=12, compaction requested=false 2024-11-11T18:15:25,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:25,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T18:15:25,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:25,077 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T18:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:25,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/f91c145826404af79743721efc893829 is 50, key is test_row_0/A:col10/1731348924293/Put/seqid=0 2024-11-11T18:15:25,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742289_1465 (size=12001) 2024-11-11T18:15:25,088 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/f91c145826404af79743721efc893829 2024-11-11T18:15:25,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/56df5676bc774e408bf093a80e1ae46a is 50, key is test_row_0/B:col10/1731348924293/Put/seqid=0 2024-11-11T18:15:25,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742290_1466 (size=12001) 2024-11-11T18:15:25,102 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/56df5676bc774e408bf093a80e1ae46a 2024-11-11T18:15:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/44adb52db9df479b871c0a7ca4e3d6d1 is 50, key is test_row_0/C:col10/1731348924293/Put/seqid=0 2024-11-11T18:15:25,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742291_1467 (size=12001) 2024-11-11T18:15:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T18:15:25,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348985383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348985389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348985396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348985464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348985465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348985504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348985504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,515 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/44adb52db9df479b871c0a7ca4e3d6d1 2024-11-11T18:15:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/f91c145826404af79743721efc893829 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f91c145826404af79743721efc893829 2024-11-11T18:15:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348985515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,522 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f91c145826404af79743721efc893829, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T18:15:25,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/56df5676bc774e408bf093a80e1ae46a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/56df5676bc774e408bf093a80e1ae46a 2024-11-11T18:15:25,526 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/56df5676bc774e408bf093a80e1ae46a, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T18:15:25,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/44adb52db9df479b871c0a7ca4e3d6d1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44adb52db9df479b871c0a7ca4e3d6d1 2024-11-11T18:15:25,530 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44adb52db9df479b871c0a7ca4e3d6d1, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T18:15:25,531 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 61e498c0de1e69a7e5cebf09f9286b5a in 454ms, sequenceid=47, compaction requested=false 2024-11-11T18:15:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:25,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T18:15:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T18:15:25,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T18:15:25,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2024-11-11T18:15:25,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3790 sec 2024-11-11T18:15:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:25,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:15:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:25,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/05c9cd9a39bd44578a2135732102057d is 50, key is test_row_0/A:col10/1731348925725/Put/seqid=0 2024-11-11T18:15:25,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742292_1468 (size=14341) 2024-11-11T18:15:25,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348985778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348985785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348985786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348985895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348985904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348985904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:25,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348986115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348986124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348986124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/05c9cd9a39bd44578a2135732102057d 2024-11-11T18:15:26,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/393887301bd7492389bc6e16ba58be77 is 50, key is test_row_0/B:col10/1731348925725/Put/seqid=0 2024-11-11T18:15:26,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742293_1469 (size=12001) 2024-11-11T18:15:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T18:15:26,315 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:26,315 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T18:15:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T18:15:26,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:26,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:26,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T18:15:26,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348986434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348986444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348986444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T18:15:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348986485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348986485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,492 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,492 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/393887301bd7492389bc6e16ba58be77 2024-11-11T18:15:26,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/665ff080293345e3a1d84a661a1cbf95 is 50, key is test_row_0/C:col10/1731348925725/Put/seqid=0 2024-11-11T18:15:26,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742294_1470 (size=12001) 2024-11-11T18:15:26,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T18:15:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T18:15:26,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T18:15:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T18:15:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T18:15:26,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348986944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/665ff080293345e3a1d84a661a1cbf95 2024-11-11T18:15:26,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/05c9cd9a39bd44578a2135732102057d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05c9cd9a39bd44578a2135732102057d 2024-11-11T18:15:26,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05c9cd9a39bd44578a2135732102057d, entries=200, sequenceid=59, filesize=14.0 K 2024-11-11T18:15:26,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/393887301bd7492389bc6e16ba58be77 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/393887301bd7492389bc6e16ba58be77 2024-11-11T18:15:26,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348986965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348986965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:26,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/393887301bd7492389bc6e16ba58be77, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T18:15:26,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/665ff080293345e3a1d84a661a1cbf95 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/665ff080293345e3a1d84a661a1cbf95 2024-11-11T18:15:26,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/665ff080293345e3a1d84a661a1cbf95, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T18:15:26,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 61e498c0de1e69a7e5cebf09f9286b5a in 1252ms, sequenceid=59, compaction requested=true 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:26,979 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:26,979 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:26,980 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:26,980 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:26,980 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:26,980 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:26,980 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,980 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:26,980 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8cd22704734a49b1929b3962c84cc680, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f91c145826404af79743721efc893829, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05c9cd9a39bd44578a2135732102057d] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=39.7 K 2024-11-11T18:15:26,980 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/10b04d99ed844118a740d1a751db15cc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/56df5676bc774e408bf093a80e1ae46a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/393887301bd7492389bc6e16ba58be77] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.2 K 2024-11-11T18:15:26,981 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd22704734a49b1929b3962c84cc680, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731348924147 2024-11-11T18:15:26,981 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b04d99ed844118a740d1a751db15cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731348924147 2024-11-11T18:15:26,981 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f91c145826404af79743721efc893829, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731348924258 2024-11-11T18:15:26,981 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 56df5676bc774e408bf093a80e1ae46a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731348924258 2024-11-11T18:15:26,981 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c9cd9a39bd44578a2135732102057d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348925382 2024-11-11T18:15:26,981 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 393887301bd7492389bc6e16ba58be77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348925382 2024-11-11T18:15:26,988 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:26,988 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/0f1b4b46327e4fbaa9854ae76baa42e1 is 50, key is test_row_0/A:col10/1731348925725/Put/seqid=0 2024-11-11T18:15:26,994 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:26,995 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/166a755e8df045e8a2ce19b65ad216c6 is 50, key is test_row_0/B:col10/1731348925725/Put/seqid=0 2024-11-11T18:15:27,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742295_1471 (size=12104) 2024-11-11T18:15:27,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742296_1472 (size=12104) 2024-11-11T18:15:27,048 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:15:27,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T18:15:27,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:27,084 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T18:15:27,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:27,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:27,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:27,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:27,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:27,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:27,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/e5068ac143554d01b6acd2c4539dc01e is 50, key is test_row_0/A:col10/1731348925783/Put/seqid=0 2024-11-11T18:15:27,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742297_1473 (size=12001) 2024-11-11T18:15:27,407 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/0f1b4b46327e4fbaa9854ae76baa42e1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0f1b4b46327e4fbaa9854ae76baa42e1 2024-11-11T18:15:27,411 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into 0f1b4b46327e4fbaa9854ae76baa42e1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:27,411 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:27,411 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=13, startTime=1731348926979; duration=0sec 2024-11-11T18:15:27,411 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:27,411 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:27,411 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:27,412 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:27,412 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:27,412 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:27,412 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44a0c5f35ad04f32b9daa4aa6e61a7e5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44adb52db9df479b871c0a7ca4e3d6d1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/665ff080293345e3a1d84a661a1cbf95] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.2 K 2024-11-11T18:15:27,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a0c5f35ad04f32b9daa4aa6e61a7e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731348924147 2024-11-11T18:15:27,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44adb52db9df479b871c0a7ca4e3d6d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731348924258 2024-11-11T18:15:27,413 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/166a755e8df045e8a2ce19b65ad216c6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/166a755e8df045e8a2ce19b65ad216c6 2024-11-11T18:15:27,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665ff080293345e3a1d84a661a1cbf95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348925382 2024-11-11T18:15:27,417 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 166a755e8df045e8a2ce19b65ad216c6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:27,417 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:27,417 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=13, startTime=1731348926979; duration=0sec 2024-11-11T18:15:27,417 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:27,417 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:27,421 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#399 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:27,421 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/df06f3be54fb4d339bdf082da3312065 is 50, key is test_row_0/C:col10/1731348925725/Put/seqid=0 2024-11-11T18:15:27,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742298_1474 (size=12104) 2024-11-11T18:15:27,428 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/df06f3be54fb4d339bdf082da3312065 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/df06f3be54fb4d339bdf082da3312065 2024-11-11T18:15:27,432 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into df06f3be54fb4d339bdf082da3312065(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:27,432 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:27,432 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=13, startTime=1731348926979; duration=0sec 2024-11-11T18:15:27,432 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:27,432 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T18:15:27,494 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/e5068ac143554d01b6acd2c4539dc01e 2024-11-11T18:15:27,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/d1cb1027684042abbd596f5d46d54a74 is 50, key is test_row_0/B:col10/1731348925783/Put/seqid=0 2024-11-11T18:15:27,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742299_1475 (size=12001) 2024-11-11T18:15:27,905 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/d1cb1027684042abbd596f5d46d54a74 2024-11-11T18:15:27,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/b88cb284319e4284882fc57014966ca9 is 50, key is test_row_0/C:col10/1731348925783/Put/seqid=0 2024-11-11T18:15:27,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742300_1476 (size=12001) 2024-11-11T18:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:27,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:28,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348987999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348988007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348988008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348988115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348988115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348988128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,320 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/b88cb284319e4284882fc57014966ca9 2024-11-11T18:15:28,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/e5068ac143554d01b6acd2c4539dc01e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/e5068ac143554d01b6acd2c4539dc01e 2024-11-11T18:15:28,327 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/e5068ac143554d01b6acd2c4539dc01e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T18:15:28,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/d1cb1027684042abbd596f5d46d54a74 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/d1cb1027684042abbd596f5d46d54a74 2024-11-11T18:15:28,332 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/d1cb1027684042abbd596f5d46d54a74, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T18:15:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/b88cb284319e4284882fc57014966ca9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/b88cb284319e4284882fc57014966ca9 2024-11-11T18:15:28,336 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/b88cb284319e4284882fc57014966ca9, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T18:15:28,337 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 61e498c0de1e69a7e5cebf09f9286b5a in 1253ms, sequenceid=79, compaction requested=false 2024-11-11T18:15:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T18:15:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T18:15:28,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T18:15:28,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-11-11T18:15:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:28,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T18:15:28,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0250 sec 2024-11-11T18:15:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:28,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/63d32c75405d4f1e83349c12addb2e92 is 50, key is test_row_0/A:col10/1731348928339/Put/seqid=0 2024-11-11T18:15:28,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742301_1477 (size=16681) 2024-11-11T18:15:28,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348988383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348988392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348988393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T18:15:28,475 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:28,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T18:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T18:15:28,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:28,478 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:28,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348988505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348988505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348988505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,512 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348988514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348988514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,526 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T18:15:28,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T18:15:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348988724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348988724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348988735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:28,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/63d32c75405d4f1e83349c12addb2e92 2024-11-11T18:15:28,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/ac04004ba39d4ac5a046a7107c71466b is 50, key is test_row_0/B:col10/1731348928339/Put/seqid=0 2024-11-11T18:15:28,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742302_1478 (size=12001) 2024-11-11T18:15:28,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T18:15:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T18:15:28,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T18:15:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:28,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:29,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348989034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348989034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348989044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T18:15:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:29,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T18:15:29,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/ac04004ba39d4ac5a046a7107c71466b 2024-11-11T18:15:29,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/865b118d15584babb24aa4fca9ed8ee8 is 50, key is test_row_0/C:col10/1731348928339/Put/seqid=0 2024-11-11T18:15:29,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742303_1479 (size=12001) 2024-11-11T18:15:29,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/865b118d15584babb24aa4fca9ed8ee8 2024-11-11T18:15:29,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/63d32c75405d4f1e83349c12addb2e92 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/63d32c75405d4f1e83349c12addb2e92 2024-11-11T18:15:29,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/63d32c75405d4f1e83349c12addb2e92, entries=250, sequenceid=96, filesize=16.3 K 2024-11-11T18:15:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/ac04004ba39d4ac5a046a7107c71466b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/ac04004ba39d4ac5a046a7107c71466b 2024-11-11T18:15:29,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/ac04004ba39d4ac5a046a7107c71466b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:15:29,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/865b118d15584babb24aa4fca9ed8ee8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/865b118d15584babb24aa4fca9ed8ee8 2024-11-11T18:15:29,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/865b118d15584babb24aa4fca9ed8ee8, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:15:29,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 61e498c0de1e69a7e5cebf09f9286b5a in 849ms, sequenceid=96, compaction requested=true 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:29,190 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:29,190 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:29,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:29,191 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:29,191 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:29,191 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:29,191 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:29,191 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:29,191 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:29,191 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0f1b4b46327e4fbaa9854ae76baa42e1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/e5068ac143554d01b6acd2c4539dc01e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/63d32c75405d4f1e83349c12addb2e92] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=39.8 K 2024-11-11T18:15:29,191 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/166a755e8df045e8a2ce19b65ad216c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/d1cb1027684042abbd596f5d46d54a74, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/ac04004ba39d4ac5a046a7107c71466b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.3 K 2024-11-11T18:15:29,192 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f1b4b46327e4fbaa9854ae76baa42e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348925382 2024-11-11T18:15:29,192 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 166a755e8df045e8a2ce19b65ad216c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348925382 2024-11-11T18:15:29,192 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5068ac143554d01b6acd2c4539dc01e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731348925769 2024-11-11T18:15:29,192 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d1cb1027684042abbd596f5d46d54a74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731348925769 2024-11-11T18:15:29,192 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d32c75405d4f1e83349c12addb2e92, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348927994 2024-11-11T18:15:29,192 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ac04004ba39d4ac5a046a7107c71466b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348927994 2024-11-11T18:15:29,201 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:29,201 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:29,202 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/12245e340739493889d9cc5105cb2721 is 50, key is test_row_0/B:col10/1731348928339/Put/seqid=0 2024-11-11T18:15:29,202 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/8b19cf64d0f14e0b81fd078b0f2595ff is 50, key is test_row_0/A:col10/1731348928339/Put/seqid=0 2024-11-11T18:15:29,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742305_1481 (size=12207) 2024-11-11T18:15:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742304_1480 (size=12207) 2024-11-11T18:15:29,217 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/8b19cf64d0f14e0b81fd078b0f2595ff as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8b19cf64d0f14e0b81fd078b0f2595ff 2024-11-11T18:15:29,223 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/12245e340739493889d9cc5105cb2721 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/12245e340739493889d9cc5105cb2721 2024-11-11T18:15:29,228 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into 8b19cf64d0f14e0b81fd078b0f2595ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:29,228 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:29,228 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=13, startTime=1731348929190; duration=0sec 2024-11-11T18:15:29,228 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:29,228 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:29,228 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:29,229 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 12245e340739493889d9cc5105cb2721(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:29,229 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:29,229 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=13, startTime=1731348929190; duration=0sec 2024-11-11T18:15:29,229 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:29,229 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:29,232 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:29,232 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:29,232 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:29,232 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/df06f3be54fb4d339bdf082da3312065, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/b88cb284319e4284882fc57014966ca9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/865b118d15584babb24aa4fca9ed8ee8] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.3 K 2024-11-11T18:15:29,233 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting df06f3be54fb4d339bdf082da3312065, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731348925382 2024-11-11T18:15:29,233 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b88cb284319e4284882fc57014966ca9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731348925769 2024-11-11T18:15:29,233 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 865b118d15584babb24aa4fca9ed8ee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348927994 2024-11-11T18:15:29,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T18:15:29,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:29,242 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T18:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:29,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/a3b19a90f95843b8a1230fb1a314b307 is 50, key is test_row_0/A:col10/1731348928370/Put/seqid=0 2024-11-11T18:15:29,250 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#407 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:29,250 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/5c5b6ba770954bd4893f918dcc074fb5 is 50, key is test_row_0/C:col10/1731348928339/Put/seqid=0 2024-11-11T18:15:29,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742307_1483 (size=12207) 2024-11-11T18:15:29,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742306_1482 (size=12001) 2024-11-11T18:15:29,288 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/a3b19a90f95843b8a1230fb1a314b307 2024-11-11T18:15:29,295 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/5c5b6ba770954bd4893f918dcc074fb5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/5c5b6ba770954bd4893f918dcc074fb5 2024-11-11T18:15:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/45d1833f79ad4de797210293e40b94a4 is 50, key is test_row_0/B:col10/1731348928370/Put/seqid=0 2024-11-11T18:15:29,303 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into 5c5b6ba770954bd4893f918dcc074fb5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:29,303 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:29,303 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=13, startTime=1731348929190; duration=0sec 2024-11-11T18:15:29,303 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:29,303 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:29,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742308_1484 (size=12001) 2024-11-11T18:15:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:29,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:29,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348989583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348989586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348989589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T18:15:29,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348989704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348989704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348989704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,714 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/45d1833f79ad4de797210293e40b94a4 2024-11-11T18:15:29,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/345c4fb637724431895ff81fbcfa74d3 is 50, key is test_row_0/C:col10/1731348928370/Put/seqid=0 2024-11-11T18:15:29,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742309_1485 (size=12001) 2024-11-11T18:15:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348989914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348989914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348989914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,125 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/345c4fb637724431895ff81fbcfa74d3 2024-11-11T18:15:30,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/a3b19a90f95843b8a1230fb1a314b307 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a3b19a90f95843b8a1230fb1a314b307 2024-11-11T18:15:30,132 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a3b19a90f95843b8a1230fb1a314b307, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T18:15:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/45d1833f79ad4de797210293e40b94a4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/45d1833f79ad4de797210293e40b94a4 2024-11-11T18:15:30,136 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/45d1833f79ad4de797210293e40b94a4, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T18:15:30,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/345c4fb637724431895ff81fbcfa74d3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/345c4fb637724431895ff81fbcfa74d3 2024-11-11T18:15:30,141 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/345c4fb637724431895ff81fbcfa74d3, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T18:15:30,142 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 61e498c0de1e69a7e5cebf09f9286b5a in 900ms, sequenceid=116, compaction requested=false 2024-11-11T18:15:30,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:30,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:30,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T18:15:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T18:15:30,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T18:15:30,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6660 sec 2024-11-11T18:15:30,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6710 sec 2024-11-11T18:15:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:30,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T18:15:30,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:30,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:30,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:30,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:30,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:30,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:30,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/f12a1eb9cd104189895b050374f09b32 is 50, key is test_row_0/A:col10/1731348930229/Put/seqid=0 2024-11-11T18:15:30,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742310_1486 (size=16831) 2024-11-11T18:15:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348990265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348990267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348990268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348990384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348990384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348990384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348990604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348990604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348990604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T18:15:30,625 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:30,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T18:15:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:30,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:30,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:30,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:30,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/f12a1eb9cd104189895b050374f09b32 2024-11-11T18:15:30,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/94d1e96a8935446b8dec2b3112193c1f is 50, key is test_row_0/B:col10/1731348930229/Put/seqid=0 2024-11-11T18:15:30,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742311_1487 (size=12101) 2024-11-11T18:15:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:30,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T18:15:30,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:30,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:30,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:30,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:30,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348990914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348990914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348990914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:30,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T18:15:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:30,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:30,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:31,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/94d1e96a8935446b8dec2b3112193c1f 2024-11-11T18:15:31,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/94828aa74b7f4945b5cd7cd1422a3e85 is 50, key is test_row_0/C:col10/1731348930229/Put/seqid=0 2024-11-11T18:15:31,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742312_1488 (size=12101) 2024-11-11T18:15:31,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T18:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T18:15:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:31,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T18:15:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:31,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348991425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:31,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348991425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348991425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:31,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:31,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/94828aa74b7f4945b5cd7cd1422a3e85 2024-11-11T18:15:31,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/f12a1eb9cd104189895b050374f09b32 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f12a1eb9cd104189895b050374f09b32 2024-11-11T18:15:31,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f12a1eb9cd104189895b050374f09b32, entries=250, sequenceid=133, filesize=16.4 K 2024-11-11T18:15:31,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/94d1e96a8935446b8dec2b3112193c1f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/94d1e96a8935446b8dec2b3112193c1f 2024-11-11T18:15:31,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/94d1e96a8935446b8dec2b3112193c1f, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T18:15:31,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/94828aa74b7f4945b5cd7cd1422a3e85 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/94828aa74b7f4945b5cd7cd1422a3e85 2024-11-11T18:15:31,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/94828aa74b7f4945b5cd7cd1422a3e85, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T18:15:31,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 61e498c0de1e69a7e5cebf09f9286b5a in 1253ms, sequenceid=133, compaction requested=true 2024-11-11T18:15:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:31,483 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:31,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:31,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:31,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:31,483 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:31,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:31,483 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:31,483 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:31,483 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:31,483 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:31,483 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,483 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,483 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/12245e340739493889d9cc5105cb2721, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/45d1833f79ad4de797210293e40b94a4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/94d1e96a8935446b8dec2b3112193c1f] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.5 K 2024-11-11T18:15:31,484 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8b19cf64d0f14e0b81fd078b0f2595ff, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a3b19a90f95843b8a1230fb1a314b307, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f12a1eb9cd104189895b050374f09b32] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=40.1 K 2024-11-11T18:15:31,484 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 12245e340739493889d9cc5105cb2721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348927994 2024-11-11T18:15:31,484 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b19cf64d0f14e0b81fd078b0f2595ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348927994 2024-11-11T18:15:31,484 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d1833f79ad4de797210293e40b94a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731348928370 2024-11-11T18:15:31,484 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b19a90f95843b8a1230fb1a314b307, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731348928370 2024-11-11T18:15:31,484 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 94d1e96a8935446b8dec2b3112193c1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348929582 2024-11-11T18:15:31,484 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12a1eb9cd104189895b050374f09b32, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348929570 2024-11-11T18:15:31,490 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:31,490 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/0b3aee9d4d6d4162a039a9b4f983b0de is 50, key is test_row_0/B:col10/1731348930229/Put/seqid=0 2024-11-11T18:15:31,491 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:31,491 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/48c629202bc541f9a472a5d3893fb095 is 50, key is test_row_0/A:col10/1731348930229/Put/seqid=0 2024-11-11T18:15:31,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742314_1490 (size=12409) 2024-11-11T18:15:31,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742313_1489 (size=12409) 2024-11-11T18:15:31,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,546 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:31,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/ae3e1abaf44b4ff8b052a67e7dbb3618 is 50, key is test_row_0/A:col10/1731348930266/Put/seqid=0 2024-11-11T18:15:31,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742315_1491 (size=12151) 2024-11-11T18:15:31,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:31,900 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/48c629202bc541f9a472a5d3893fb095 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/48c629202bc541f9a472a5d3893fb095 2024-11-11T18:15:31,900 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/0b3aee9d4d6d4162a039a9b4f983b0de as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/0b3aee9d4d6d4162a039a9b4f983b0de 2024-11-11T18:15:31,905 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into 48c629202bc541f9a472a5d3893fb095(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:31,905 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 0b3aee9d4d6d4162a039a9b4f983b0de(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:31,905 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=13, startTime=1731348931482; duration=0sec 2024-11-11T18:15:31,905 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=13, startTime=1731348931483; duration=0sec 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:31,905 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:31,906 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:31,906 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:31,906 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:31,906 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/5c5b6ba770954bd4893f918dcc074fb5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/345c4fb637724431895ff81fbcfa74d3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/94828aa74b7f4945b5cd7cd1422a3e85] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.5 K 2024-11-11T18:15:31,906 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5b6ba770954bd4893f918dcc074fb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348927994 2024-11-11T18:15:31,906 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345c4fb637724431895ff81fbcfa74d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731348928370 2024-11-11T18:15:31,907 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94828aa74b7f4945b5cd7cd1422a3e85, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348929582 2024-11-11T18:15:31,912 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:31,912 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/4894b11834ba49e48a96c03cb3dc63da is 50, key is test_row_0/C:col10/1731348930229/Put/seqid=0 2024-11-11T18:15:31,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742316_1492 (size=12409) 2024-11-11T18:15:31,954 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/ae3e1abaf44b4ff8b052a67e7dbb3618 2024-11-11T18:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/274184745e57423fb6234d6b20f8d4ef is 50, key is test_row_0/B:col10/1731348930266/Put/seqid=0 2024-11-11T18:15:31,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742317_1493 (size=12151) 2024-11-11T18:15:32,329 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/4894b11834ba49e48a96c03cb3dc63da as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/4894b11834ba49e48a96c03cb3dc63da 2024-11-11T18:15:32,338 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into 4894b11834ba49e48a96c03cb3dc63da(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:32,338 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:32,338 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=13, startTime=1731348931483; duration=0sec 2024-11-11T18:15:32,338 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:32,338 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:32,369 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/274184745e57423fb6234d6b20f8d4ef 2024-11-11T18:15:32,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/abc29a6e124045ac877bae57ea10baaa is 50, key is test_row_0/C:col10/1731348930266/Put/seqid=0 2024-11-11T18:15:32,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742318_1494 (size=12151) 2024-11-11T18:15:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:32,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348992475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348992478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348992482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731348992524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,535 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731348992534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,543 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348992594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348992594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348992604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:32,785 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/abc29a6e124045ac877bae57ea10baaa 2024-11-11T18:15:32,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/ae3e1abaf44b4ff8b052a67e7dbb3618 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/ae3e1abaf44b4ff8b052a67e7dbb3618 2024-11-11T18:15:32,792 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/ae3e1abaf44b4ff8b052a67e7dbb3618, entries=150, sequenceid=149, filesize=11.9 K 2024-11-11T18:15:32,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/274184745e57423fb6234d6b20f8d4ef as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/274184745e57423fb6234d6b20f8d4ef 2024-11-11T18:15:32,796 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/274184745e57423fb6234d6b20f8d4ef, entries=150, sequenceid=149, filesize=11.9 K 2024-11-11T18:15:32,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/abc29a6e124045ac877bae57ea10baaa as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/abc29a6e124045ac877bae57ea10baaa 2024-11-11T18:15:32,799 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/abc29a6e124045ac877bae57ea10baaa, entries=150, sequenceid=149, filesize=11.9 K 2024-11-11T18:15:32,800 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 61e498c0de1e69a7e5cebf09f9286b5a in 1254ms, sequenceid=149, compaction requested=false 2024-11-11T18:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:32,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T18:15:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T18:15:32,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T18:15:32,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1740 sec 2024-11-11T18:15:32,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1780 sec 2024-11-11T18:15:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:32,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T18:15:32,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:32,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:32,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:32,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:32,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:32,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:32,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/2aa5e949c3614b529fdf6e39e4c2c1e6 is 50, key is test_row_0/A:col10/1731348932474/Put/seqid=0 2024-11-11T18:15:32,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742319_1495 (size=14541) 2024-11-11T18:15:32,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348992853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348992858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,870 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348992868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348992975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348992975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:32,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348992984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348993194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:33,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348993195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348993195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/2aa5e949c3614b529fdf6e39e4c2c1e6 2024-11-11T18:15:33,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/2b66d6574eac4c448e8e5ae1d93b8430 is 50, key is test_row_0/B:col10/1731348932474/Put/seqid=0 2024-11-11T18:15:33,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742320_1496 (size=12151) 2024-11-11T18:15:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348993514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348993514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348993514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:33,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/2b66d6574eac4c448e8e5ae1d93b8430 2024-11-11T18:15:33,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/287e530a53e9464898eb67c14ff2626a is 50, key is test_row_0/C:col10/1731348932474/Put/seqid=0 2024-11-11T18:15:33,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742321_1497 (size=12151) 2024-11-11T18:15:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348994034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348994034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:34,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348994034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:34,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/287e530a53e9464898eb67c14ff2626a 2024-11-11T18:15:34,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/2aa5e949c3614b529fdf6e39e4c2c1e6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/2aa5e949c3614b529fdf6e39e4c2c1e6 2024-11-11T18:15:34,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/2aa5e949c3614b529fdf6e39e4c2c1e6, entries=200, sequenceid=169, filesize=14.2 K 2024-11-11T18:15:34,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/2b66d6574eac4c448e8e5ae1d93b8430 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/2b66d6574eac4c448e8e5ae1d93b8430 2024-11-11T18:15:34,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/2b66d6574eac4c448e8e5ae1d93b8430, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T18:15:34,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/287e530a53e9464898eb67c14ff2626a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/287e530a53e9464898eb67c14ff2626a 2024-11-11T18:15:34,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/287e530a53e9464898eb67c14ff2626a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T18:15:34,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 61e498c0de1e69a7e5cebf09f9286b5a in 1245ms, sequenceid=169, compaction requested=true 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:34,066 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:34,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:34,066 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:34,066 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:34,066 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:34,067 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:34,067 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:34,067 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/48c629202bc541f9a472a5d3893fb095, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/ae3e1abaf44b4ff8b052a67e7dbb3618, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/2aa5e949c3614b529fdf6e39e4c2c1e6] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=38.2 K 2024-11-11T18:15:34,067 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/0b3aee9d4d6d4162a039a9b4f983b0de, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/274184745e57423fb6234d6b20f8d4ef, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/2b66d6574eac4c448e8e5ae1d93b8430] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.9 K 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c629202bc541f9a472a5d3893fb095, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348929582 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b3aee9d4d6d4162a039a9b4f983b0de, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348929582 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3e1abaf44b4ff8b052a67e7dbb3618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731348930253 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 274184745e57423fb6234d6b20f8d4ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731348930253 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa5e949c3614b529fdf6e39e4c2c1e6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731348932474 2024-11-11T18:15:34,067 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b66d6574eac4c448e8e5ae1d93b8430, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731348932474 2024-11-11T18:15:34,080 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:34,080 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/9aa5175d991d4bd8a02b3d02b5c682a4 is 50, key is test_row_0/B:col10/1731348932474/Put/seqid=0 2024-11-11T18:15:34,082 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:34,083 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/d5f236e4257c4e51b92e8a72efada05f is 50, key is test_row_0/A:col10/1731348932474/Put/seqid=0 2024-11-11T18:15:34,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742322_1498 (size=12561) 2024-11-11T18:15:34,103 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/9aa5175d991d4bd8a02b3d02b5c682a4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/9aa5175d991d4bd8a02b3d02b5c682a4 2024-11-11T18:15:34,107 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 9aa5175d991d4bd8a02b3d02b5c682a4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:34,107 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:34,108 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=13, startTime=1731348934066; duration=0sec 2024-11-11T18:15:34,108 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:34,108 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:34,108 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:34,108 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:34,109 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:34,109 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:34,109 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/4894b11834ba49e48a96c03cb3dc63da, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/abc29a6e124045ac877bae57ea10baaa, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/287e530a53e9464898eb67c14ff2626a] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=35.9 K 2024-11-11T18:15:34,109 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 4894b11834ba49e48a96c03cb3dc63da, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348929582 2024-11-11T18:15:34,110 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting abc29a6e124045ac877bae57ea10baaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731348930253 2024-11-11T18:15:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742323_1499 (size=12561) 2024-11-11T18:15:34,110 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 287e530a53e9464898eb67c14ff2626a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731348932474 2024-11-11T18:15:34,115 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/d5f236e4257c4e51b92e8a72efada05f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/d5f236e4257c4e51b92e8a72efada05f 2024-11-11T18:15:34,121 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into d5f236e4257c4e51b92e8a72efada05f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:34,121 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:34,121 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=13, startTime=1731348934066; duration=0sec 2024-11-11T18:15:34,121 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:34,121 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:34,124 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:34,125 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7bd9751ff95e4a6c9dcc1940e71b2760 is 50, key is test_row_0/C:col10/1731348932474/Put/seqid=0 2024-11-11T18:15:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742324_1500 (size=12561) 2024-11-11T18:15:34,145 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7bd9751ff95e4a6c9dcc1940e71b2760 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7bd9751ff95e4a6c9dcc1940e71b2760 2024-11-11T18:15:34,149 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into 7bd9751ff95e4a6c9dcc1940e71b2760(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:34,149 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:34,149 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=13, startTime=1731348934066; duration=0sec 2024-11-11T18:15:34,149 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:34,150 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T18:15:34,795 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:34,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-11T18:15:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T18:15:34,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:34,798 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:34,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T18:15:34,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T18:15:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:34,950 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T18:15:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:34,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/358e0d0af2944b96a86edc3529b8446d is 50, key is test_row_0/A:col10/1731348932866/Put/seqid=0 2024-11-11T18:15:34,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742325_1501 (size=12151) 2024-11-11T18:15:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:35,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348995091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348995092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348995094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T18:15:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348995214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348995215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348995215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,360 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/358e0d0af2944b96a86edc3529b8446d 2024-11-11T18:15:35,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/7120a9dbd89a4c7db4476143983ceccf is 50, key is test_row_0/B:col10/1731348932866/Put/seqid=0 2024-11-11T18:15:35,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742326_1502 (size=12151) 2024-11-11T18:15:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T18:15:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348995435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348995435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348995435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348995744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348995744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348995744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:35,772 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/7120a9dbd89a4c7db4476143983ceccf 2024-11-11T18:15:35,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/264755dbbece4377bd7f3835f03b9465 is 50, key is test_row_0/C:col10/1731348932866/Put/seqid=0 2024-11-11T18:15:35,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742327_1503 (size=12151) 2024-11-11T18:15:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T18:15:36,185 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/264755dbbece4377bd7f3835f03b9465 2024-11-11T18:15:36,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/358e0d0af2944b96a86edc3529b8446d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/358e0d0af2944b96a86edc3529b8446d 2024-11-11T18:15:36,193 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/358e0d0af2944b96a86edc3529b8446d, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T18:15:36,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/7120a9dbd89a4c7db4476143983ceccf as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7120a9dbd89a4c7db4476143983ceccf 2024-11-11T18:15:36,196 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7120a9dbd89a4c7db4476143983ceccf, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T18:15:36,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/264755dbbece4377bd7f3835f03b9465 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/264755dbbece4377bd7f3835f03b9465 2024-11-11T18:15:36,200 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/264755dbbece4377bd7f3835f03b9465, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T18:15:36,201 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 61e498c0de1e69a7e5cebf09f9286b5a in 1251ms, sequenceid=187, compaction requested=false 2024-11-11T18:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:36,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-11T18:15:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-11T18:15:36,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T18:15:36,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-11T18:15:36,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4090 sec 2024-11-11T18:15:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:36,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T18:15:36,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:36,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:36,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/b46fc24c3ca94285ae77809837727a9d is 50, key is test_row_0/A:col10/1731348936273/Put/seqid=0 2024-11-11T18:15:36,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742328_1504 (size=14541) 2024-11-11T18:15:36,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348996305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348996313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348996313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348996424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348996434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348996434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348996644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348996644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348996644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:36,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/b46fc24c3ca94285ae77809837727a9d 2024-11-11T18:15:36,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/09473bed120d40af83878ca200bc0cba is 50, key is test_row_0/B:col10/1731348936273/Put/seqid=0 2024-11-11T18:15:36,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742329_1505 (size=12151) 2024-11-11T18:15:36,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/09473bed120d40af83878ca200bc0cba 2024-11-11T18:15:36,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/709119fb3a2f4e75bb9cc5f286677e60 is 50, key is test_row_0/C:col10/1731348936273/Put/seqid=0 2024-11-11T18:15:36,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742330_1506 (size=12151) 2024-11-11T18:15:36,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/709119fb3a2f4e75bb9cc5f286677e60 2024-11-11T18:15:36,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/b46fc24c3ca94285ae77809837727a9d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/b46fc24c3ca94285ae77809837727a9d 2024-11-11T18:15:36,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/b46fc24c3ca94285ae77809837727a9d, entries=200, sequenceid=206, filesize=14.2 K 2024-11-11T18:15:36,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/09473bed120d40af83878ca200bc0cba as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/09473bed120d40af83878ca200bc0cba 2024-11-11T18:15:36,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/09473bed120d40af83878ca200bc0cba, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T18:15:36,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/709119fb3a2f4e75bb9cc5f286677e60 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/709119fb3a2f4e75bb9cc5f286677e60 2024-11-11T18:15:36,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/709119fb3a2f4e75bb9cc5f286677e60, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T18:15:36,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 61e498c0de1e69a7e5cebf09f9286b5a in 472ms, sequenceid=206, compaction requested=true 2024-11-11T18:15:36,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:36,745 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:36,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:36,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:36,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:36,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:36,746 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:36,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:36,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:36,746 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:36,746 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:36,746 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:36,746 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:36,746 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/d5f236e4257c4e51b92e8a72efada05f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/358e0d0af2944b96a86edc3529b8446d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/b46fc24c3ca94285ae77809837727a9d] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=38.3 K 2024-11-11T18:15:36,746 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:36,746 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:36,746 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/9aa5175d991d4bd8a02b3d02b5c682a4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7120a9dbd89a4c7db4476143983ceccf, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/09473bed120d40af83878ca200bc0cba] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=36.0 K 2024-11-11T18:15:36,746 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f236e4257c4e51b92e8a72efada05f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731348932474 2024-11-11T18:15:36,747 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa5175d991d4bd8a02b3d02b5c682a4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731348932474 2024-11-11T18:15:36,747 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 358e0d0af2944b96a86edc3529b8446d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731348932835 2024-11-11T18:15:36,747 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7120a9dbd89a4c7db4476143983ceccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731348932835 2024-11-11T18:15:36,748 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 09473bed120d40af83878ca200bc0cba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731348935079 2024-11-11T18:15:36,748 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b46fc24c3ca94285ae77809837727a9d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731348935079 2024-11-11T18:15:36,754 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:36,754 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/870385ac6ec24960a5748c8829341427 is 50, key is test_row_0/B:col10/1731348936273/Put/seqid=0 2024-11-11T18:15:36,756 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:36,757 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/84e537bae7ef4b22ac61c669d31ac25e is 50, key is test_row_0/A:col10/1731348936273/Put/seqid=0 2024-11-11T18:15:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742331_1507 (size=12663) 2024-11-11T18:15:36,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742332_1508 (size=12663) 2024-11-11T18:15:36,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T18:15:36,945 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:36,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-11T18:15:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:36,947 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:36,948 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:36,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:36,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T18:15:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:36,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:36,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/08a00a87b10b4a70a89069a1b4754045 is 50, key is test_row_0/A:col10/1731348936960/Put/seqid=0 2024-11-11T18:15:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742333_1509 (size=14541) 2024-11-11T18:15:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348997009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348997011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348997011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:37,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348997135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348997135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348997135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,166 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/870385ac6ec24960a5748c8829341427 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/870385ac6ec24960a5748c8829341427 2024-11-11T18:15:37,169 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/84e537bae7ef4b22ac61c669d31ac25e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/84e537bae7ef4b22ac61c669d31ac25e 2024-11-11T18:15:37,170 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 870385ac6ec24960a5748c8829341427(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:37,170 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:37,170 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=13, startTime=1731348936745; duration=0sec 2024-11-11T18:15:37,170 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:37,170 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:37,170 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:37,171 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:37,172 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:37,172 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,172 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7bd9751ff95e4a6c9dcc1940e71b2760, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/264755dbbece4377bd7f3835f03b9465, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/709119fb3a2f4e75bb9cc5f286677e60] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=36.0 K 2024-11-11T18:15:37,172 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd9751ff95e4a6c9dcc1940e71b2760, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731348932474 2024-11-11T18:15:37,172 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 264755dbbece4377bd7f3835f03b9465, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731348932835 2024-11-11T18:15:37,173 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 709119fb3a2f4e75bb9cc5f286677e60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731348935079 2024-11-11T18:15:37,174 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into 84e537bae7ef4b22ac61c669d31ac25e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:37,174 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:37,174 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=13, startTime=1731348936745; duration=0sec 2024-11-11T18:15:37,174 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:37,175 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:37,179 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:37,180 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/a05c3bb19c36457fa2ec92c6f771e7ce is 50, key is test_row_0/C:col10/1731348936273/Put/seqid=0 2024-11-11T18:15:37,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742334_1510 (size=12663) 2024-11-11T18:15:37,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348997355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348997355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348997355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/08a00a87b10b4a70a89069a1b4754045 2024-11-11T18:15:37,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/15d0dcaadbd04c11b744e6d6ab842128 is 50, key is test_row_0/B:col10/1731348936960/Put/seqid=0 2024-11-11T18:15:37,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742335_1511 (size=12151) 2024-11-11T18:15:37,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:37,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:37,587 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/a05c3bb19c36457fa2ec92c6f771e7ce as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/a05c3bb19c36457fa2ec92c6f771e7ce 2024-11-11T18:15:37,591 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into a05c3bb19c36457fa2ec92c6f771e7ce(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:37,591 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:37,591 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=13, startTime=1731348936745; duration=0sec 2024-11-11T18:15:37,591 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:37,591 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:37,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348997674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348997675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348997675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:37,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/15d0dcaadbd04c11b744e6d6ab842128 2024-11-11T18:15:37,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/486de68295d54e849fd078b21f9eaef9 is 50, key is test_row_0/C:col10/1731348936960/Put/seqid=0 2024-11-11T18:15:37,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742336_1512 (size=12151) 2024-11-11T18:15:37,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:37,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:37,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:38,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:38,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:38,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:38,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:38,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:38,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:38,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/486de68295d54e849fd078b21f9eaef9 2024-11-11T18:15:38,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/08a00a87b10b4a70a89069a1b4754045 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/08a00a87b10b4a70a89069a1b4754045 2024-11-11T18:15:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348998195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:38,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/08a00a87b10b4a70a89069a1b4754045, entries=200, sequenceid=221, filesize=14.2 K 2024-11-11T18:15:38,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348998195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:38,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348998195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:38,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:38,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/15d0dcaadbd04c11b744e6d6ab842128 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/15d0dcaadbd04c11b744e6d6ab842128 2024-11-11T18:15:38,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/15d0dcaadbd04c11b744e6d6ab842128, entries=150, sequenceid=221, filesize=11.9 K 2024-11-11T18:15:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/486de68295d54e849fd078b21f9eaef9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/486de68295d54e849fd078b21f9eaef9 2024-11-11T18:15:38,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/486de68295d54e849fd078b21f9eaef9, entries=150, sequenceid=221, filesize=11.9 K 2024-11-11T18:15:38,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 61e498c0de1e69a7e5cebf09f9286b5a in 1248ms, sequenceid=221, compaction requested=false 2024-11-11T18:15:38,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:38,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T18:15:38,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:38,323 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T18:15:38,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:38,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/83cae472e392459fb3f4d6c3b1602d12 is 50, key is test_row_0/A:col10/1731348936997/Put/seqid=0 2024-11-11T18:15:38,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742337_1513 (size=12151) 2024-11-11T18:15:38,738 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/83cae472e392459fb3f4d6c3b1602d12 2024-11-11T18:15:38,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/bae4b477e6d1471f9bd40637f383d48c is 50, key is test_row_0/B:col10/1731348936997/Put/seqid=0 2024-11-11T18:15:38,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742338_1514 (size=12151) 2024-11-11T18:15:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:39,151 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/bae4b477e6d1471f9bd40637f383d48c 2024-11-11T18:15:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/c573481f62e144bcb67ffd394c56275a is 50, key is test_row_0/C:col10/1731348936997/Put/seqid=0 2024-11-11T18:15:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742339_1515 (size=12151) 2024-11-11T18:15:39,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:39,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348999249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348999251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348999257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348999365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348999375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348999375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348999574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,580 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/c573481f62e144bcb67ffd394c56275a 2024-11-11T18:15:39,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/83cae472e392459fb3f4d6c3b1602d12 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/83cae472e392459fb3f4d6c3b1602d12 2024-11-11T18:15:39,587 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/83cae472e392459fb3f4d6c3b1602d12, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T18:15:39,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/bae4b477e6d1471f9bd40637f383d48c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/bae4b477e6d1471f9bd40637f383d48c 2024-11-11T18:15:39,591 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/bae4b477e6d1471f9bd40637f383d48c, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T18:15:39,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/c573481f62e144bcb67ffd394c56275a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/c573481f62e144bcb67ffd394c56275a 2024-11-11T18:15:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348999585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348999585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,595 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/c573481f62e144bcb67ffd394c56275a, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T18:15:39,596 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 61e498c0de1e69a7e5cebf09f9286b5a in 1272ms, sequenceid=241, compaction requested=true 2024-11-11T18:15:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-11T18:15:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-11T18:15:39,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T18:15:39,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6490 sec 2024-11-11T18:15:39,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.6540 sec 2024-11-11T18:15:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:39,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T18:15:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:39,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:39,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/42cb89153f564178a34e53a2bf9f888e is 50, key is test_row_0/A:col10/1731348939890/Put/seqid=0 2024-11-11T18:15:39,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742340_1516 (size=14591) 2024-11-11T18:15:39,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731348999945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731348999947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731348999948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349000055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349000064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349000064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349000275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349000284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349000284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/42cb89153f564178a34e53a2bf9f888e 2024-11-11T18:15:40,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/6b2332b6deeb4d10af53d59d3ebc9845 is 50, key is test_row_0/B:col10/1731348939890/Put/seqid=0 2024-11-11T18:15:40,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742341_1517 (size=12201) 2024-11-11T18:15:40,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349000594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349000605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349000605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:40,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:40,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/6b2332b6deeb4d10af53d59d3ebc9845 2024-11-11T18:15:40,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/843a296c30cd4760aa7d3b84ef26bd09 is 50, key is test_row_0/C:col10/1731348939890/Put/seqid=0 2024-11-11T18:15:40,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742342_1518 (size=12201) 2024-11-11T18:15:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T18:15:41,115 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:41,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-11T18:15:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T18:15:41,117 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:41,117 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:41,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:41,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/843a296c30cd4760aa7d3b84ef26bd09 2024-11-11T18:15:41,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:41,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349001115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349001124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:41,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349001124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:41,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:41,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:41,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/42cb89153f564178a34e53a2bf9f888e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/42cb89153f564178a34e53a2bf9f888e 2024-11-11T18:15:41,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/42cb89153f564178a34e53a2bf9f888e, entries=200, sequenceid=257, filesize=14.2 K 2024-11-11T18:15:41,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/6b2332b6deeb4d10af53d59d3ebc9845 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/6b2332b6deeb4d10af53d59d3ebc9845 2024-11-11T18:15:41,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/6b2332b6deeb4d10af53d59d3ebc9845, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T18:15:41,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/843a296c30cd4760aa7d3b84ef26bd09 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/843a296c30cd4760aa7d3b84ef26bd09 2024-11-11T18:15:41,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/843a296c30cd4760aa7d3b84ef26bd09, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T18:15:41,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 61e498c0de1e69a7e5cebf09f9286b5a in 1246ms, sequenceid=257, compaction requested=true 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:41,138 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:15:41,138 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:41,139 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:15:41,139 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:15:41,139 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:41,139 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:41,139 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:41,139 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:41,139 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/870385ac6ec24960a5748c8829341427, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/15d0dcaadbd04c11b744e6d6ab842128, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/bae4b477e6d1471f9bd40637f383d48c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/6b2332b6deeb4d10af53d59d3ebc9845] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=48.0 K 2024-11-11T18:15:41,139 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/84e537bae7ef4b22ac61c669d31ac25e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/08a00a87b10b4a70a89069a1b4754045, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/83cae472e392459fb3f4d6c3b1602d12, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/42cb89153f564178a34e53a2bf9f888e] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=52.7 K 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 870385ac6ec24960a5748c8829341427, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731348935079 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e537bae7ef4b22ac61c669d31ac25e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731348935079 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d0dcaadbd04c11b744e6d6ab842128, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731348936311 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08a00a87b10b4a70a89069a1b4754045, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731348936303 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83cae472e392459fb3f4d6c3b1602d12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731348936995 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting bae4b477e6d1471f9bd40637f383d48c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731348936995 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42cb89153f564178a34e53a2bf9f888e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731348939225 2024-11-11T18:15:41,140 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b2332b6deeb4d10af53d59d3ebc9845, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731348939249 2024-11-11T18:15:41,156 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:41,156 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:41,157 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/7df189bed7fc46fbb2c4592aec8a97bb is 50, key is test_row_0/B:col10/1731348939890/Put/seqid=0 2024-11-11T18:15:41,157 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/05be404eed4e430bb766e8a47f89ec5a is 50, key is test_row_0/A:col10/1731348939890/Put/seqid=0 2024-11-11T18:15:41,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742344_1520 (size=12849) 2024-11-11T18:15:41,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742343_1519 (size=12849) 2024-11-11T18:15:41,167 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/05be404eed4e430bb766e8a47f89ec5a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05be404eed4e430bb766e8a47f89ec5a 2024-11-11T18:15:41,172 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into 05be404eed4e430bb766e8a47f89ec5a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:41,172 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:41,172 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=12, startTime=1731348941138; duration=0sec 2024-11-11T18:15:41,172 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:41,172 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:41,172 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:15:41,173 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:15:41,173 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:41,173 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:41,173 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/a05c3bb19c36457fa2ec92c6f771e7ce, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/486de68295d54e849fd078b21f9eaef9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/c573481f62e144bcb67ffd394c56275a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/843a296c30cd4760aa7d3b84ef26bd09] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=48.0 K 2024-11-11T18:15:41,174 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting a05c3bb19c36457fa2ec92c6f771e7ce, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731348935079 2024-11-11T18:15:41,174 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 486de68295d54e849fd078b21f9eaef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731348936311 2024-11-11T18:15:41,174 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c573481f62e144bcb67ffd394c56275a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731348936995 2024-11-11T18:15:41,175 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 843a296c30cd4760aa7d3b84ef26bd09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731348939249 2024-11-11T18:15:41,181 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:41,181 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7c55bc7d0a8c45cf966ffd0a9401697f is 50, key is test_row_0/C:col10/1731348939890/Put/seqid=0 2024-11-11T18:15:41,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742345_1521 (size=12849) 2024-11-11T18:15:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T18:15:41,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T18:15:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:41,270 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T18:15:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:41,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/45b1716ed08744ea89485ed91a194480 is 50, key is test_row_0/A:col10/1731348939943/Put/seqid=0 2024-11-11T18:15:41,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742346_1522 (size=12301) 2024-11-11T18:15:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T18:15:41,569 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/7df189bed7fc46fbb2c4592aec8a97bb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7df189bed7fc46fbb2c4592aec8a97bb 2024-11-11T18:15:41,572 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 7df189bed7fc46fbb2c4592aec8a97bb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:41,573 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:41,573 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=12, startTime=1731348941138; duration=0sec 2024-11-11T18:15:41,573 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:41,573 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:41,588 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7c55bc7d0a8c45cf966ffd0a9401697f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7c55bc7d0a8c45cf966ffd0a9401697f 2024-11-11T18:15:41,594 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into 7c55bc7d0a8c45cf966ffd0a9401697f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:41,594 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:41,594 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=12, startTime=1731348941138; duration=0sec 2024-11-11T18:15:41,594 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:41,594 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:41,679 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/45b1716ed08744ea89485ed91a194480 2024-11-11T18:15:41,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/b24ff12bd6e44e948d01d980e0f195b1 is 50, key is test_row_0/B:col10/1731348939943/Put/seqid=0 2024-11-11T18:15:41,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742347_1523 (size=12301) 2024-11-11T18:15:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T18:15:42,090 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/b24ff12bd6e44e948d01d980e0f195b1 2024-11-11T18:15:42,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/e78a42c6573c4172b6a5a9918b99069c is 50, key is test_row_0/C:col10/1731348939943/Put/seqid=0 2024-11-11T18:15:42,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742348_1524 (size=12301) 2024-11-11T18:15:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:42,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:42,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349002206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349002207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349002208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T18:15:42,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349002314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349002325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349002325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,503 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/e78a42c6573c4172b6a5a9918b99069c 2024-11-11T18:15:42,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/45b1716ed08744ea89485ed91a194480 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/45b1716ed08744ea89485ed91a194480 2024-11-11T18:15:42,511 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/45b1716ed08744ea89485ed91a194480, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T18:15:42,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/b24ff12bd6e44e948d01d980e0f195b1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/b24ff12bd6e44e948d01d980e0f195b1 2024-11-11T18:15:42,515 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/b24ff12bd6e44e948d01d980e0f195b1, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T18:15:42,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/e78a42c6573c4172b6a5a9918b99069c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/e78a42c6573c4172b6a5a9918b99069c 2024-11-11T18:15:42,518 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/e78a42c6573c4172b6a5a9918b99069c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T18:15:42,519 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 61e498c0de1e69a7e5cebf09f9286b5a in 1249ms, sequenceid=274, compaction requested=false 2024-11-11T18:15:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-11T18:15:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-11T18:15:42,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T18:15:42,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-11T18:15:42,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4090 sec 2024-11-11T18:15:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:42,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T18:15:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:42,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:42,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/0de61a0256d144d8bd82f3be6a6f2f5d is 50, key is test_row_0/A:col10/1731348942186/Put/seqid=0 2024-11-11T18:15:42,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742349_1525 (size=14741) 2024-11-11T18:15:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349002557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349002566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349002567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731349002568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60754 deadline: 1731349002604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,611 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in 61e498c0de1e69a7e5cebf09f9286b5a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18332 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:15:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349002674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349002674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349002685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731349002685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349002894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349002894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349002895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731349002904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:42,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/0de61a0256d144d8bd82f3be6a6f2f5d 2024-11-11T18:15:42,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/a5e1bd57f81941d2bb67a6538f136781 is 50, key is test_row_0/B:col10/1731348942186/Put/seqid=0 2024-11-11T18:15:42,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742350_1526 (size=12301) 2024-11-11T18:15:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349003204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349003204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349003214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731349003225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T18:15:43,264 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:43,265 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-11T18:15:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T18:15:43,267 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:43,268 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:43,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:43,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/a5e1bd57f81941d2bb67a6538f136781 2024-11-11T18:15:43,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/0f16dc538a3e4c8e8d10bbf1025c0193 is 50, key is test_row_0/C:col10/1731348942186/Put/seqid=0 2024-11-11T18:15:43,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742351_1527 (size=12301) 2024-11-11T18:15:43,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T18:15:43,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T18:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T18:15:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T18:15:43,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T18:15:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60838 deadline: 1731349003725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60784 deadline: 1731349003725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731349003734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1731349003734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 , the old value is region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., hostname=90f34c5bee6a,42957,1731348800579, seqNum=2 because the exception is null or not the one we care about 2024-11-11T18:15:43,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/0f16dc538a3e4c8e8d10bbf1025c0193 2024-11-11T18:15:43,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/0de61a0256d144d8bd82f3be6a6f2f5d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0de61a0256d144d8bd82f3be6a6f2f5d 2024-11-11T18:15:43,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0de61a0256d144d8bd82f3be6a6f2f5d, entries=200, sequenceid=293, filesize=14.4 K 2024-11-11T18:15:43,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/a5e1bd57f81941d2bb67a6538f136781 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/a5e1bd57f81941d2bb67a6538f136781 2024-11-11T18:15:43,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/a5e1bd57f81941d2bb67a6538f136781, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T18:15:43,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/0f16dc538a3e4c8e8d10bbf1025c0193 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0f16dc538a3e4c8e8d10bbf1025c0193 2024-11-11T18:15:43,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0f16dc538a3e4c8e8d10bbf1025c0193, entries=150, sequenceid=293, filesize=12.0 K 2024-11-11T18:15:43,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 61e498c0de1e69a7e5cebf09f9286b5a in 1245ms, sequenceid=293, compaction requested=true 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:43,775 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:43,775 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 61e498c0de1e69a7e5cebf09f9286b5a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:15:43,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:43,776 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:43,776 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:43,776 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/A is initiating minor compaction (all files) 2024-11-11T18:15:43,776 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/B is initiating minor compaction (all files) 2024-11-11T18:15:43,776 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/B in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,776 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/A in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,776 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05be404eed4e430bb766e8a47f89ec5a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/45b1716ed08744ea89485ed91a194480, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0de61a0256d144d8bd82f3be6a6f2f5d] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=39.0 K 2024-11-11T18:15:43,776 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7df189bed7fc46fbb2c4592aec8a97bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/b24ff12bd6e44e948d01d980e0f195b1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/a5e1bd57f81941d2bb67a6538f136781] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=36.6 K 2024-11-11T18:15:43,776 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df189bed7fc46fbb2c4592aec8a97bb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731348939249 2024-11-11T18:15:43,776 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05be404eed4e430bb766e8a47f89ec5a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731348939249 2024-11-11T18:15:43,777 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b1716ed08744ea89485ed91a194480, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348939943 2024-11-11T18:15:43,777 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b24ff12bd6e44e948d01d980e0f195b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348939943 2024-11-11T18:15:43,777 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de61a0256d144d8bd82f3be6a6f2f5d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731348942186 2024-11-11T18:15:43,777 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e1bd57f81941d2bb67a6538f136781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731348942186 2024-11-11T18:15:43,784 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:43,785 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/33b86f5464284b5494d3a86961f2e173 is 50, key is test_row_0/B:col10/1731348942186/Put/seqid=0 2024-11-11T18:15:43,787 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:43,788 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/4cd31eab25ab463aa0fc237990bf66a2 is 50, key is test_row_0/A:col10/1731348942186/Put/seqid=0 2024-11-11T18:15:43,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742352_1528 (size=13051) 2024-11-11T18:15:43,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742353_1529 (size=13051) 2024-11-11T18:15:43,801 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/4cd31eab25ab463aa0fc237990bf66a2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/4cd31eab25ab463aa0fc237990bf66a2 2024-11-11T18:15:43,806 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/A of 61e498c0de1e69a7e5cebf09f9286b5a into 4cd31eab25ab463aa0fc237990bf66a2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:43,806 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:43,806 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/A, priority=13, startTime=1731348943775; duration=0sec 2024-11-11T18:15:43,806 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:15:43,806 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:A 2024-11-11T18:15:43,806 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:15:43,807 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:15:43,807 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): 61e498c0de1e69a7e5cebf09f9286b5a/C is initiating minor compaction (all files) 2024-11-11T18:15:43,807 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 61e498c0de1e69a7e5cebf09f9286b5a/C in TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,807 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7c55bc7d0a8c45cf966ffd0a9401697f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/e78a42c6573c4172b6a5a9918b99069c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0f16dc538a3e4c8e8d10bbf1025c0193] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp, totalSize=36.6 K 2024-11-11T18:15:43,807 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c55bc7d0a8c45cf966ffd0a9401697f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731348939249 2024-11-11T18:15:43,807 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting e78a42c6573c4172b6a5a9918b99069c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731348939943 2024-11-11T18:15:43,807 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f16dc538a3e4c8e8d10bbf1025c0193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731348942186 2024-11-11T18:15:43,813 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 61e498c0de1e69a7e5cebf09f9286b5a#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:15:43,813 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/0e993eea304c4ebeacd62036cfdf9bc1 is 50, key is test_row_0/C:col10/1731348942186/Put/seqid=0 2024-11-11T18:15:43,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742354_1530 (size=13051) 2024-11-11T18:15:43,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T18:15:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:43,880 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T18:15:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:43,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:43,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/a6c1490abef34c7db585d612591dfbce is 50, key is test_row_0/A:col10/1731348942565/Put/seqid=0 2024-11-11T18:15:43,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742355_1531 (size=12301) 2024-11-11T18:15:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T18:15:44,148 INFO [Thread-2075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2075. 2024-11-11T18:15:44,149 DEBUG [Thread-2075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,149 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,149 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,150 INFO [Thread-2077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2077. 2024-11-11T18:15:44,150 DEBUG [Thread-2077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,150 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,150 DEBUG [Thread-2077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,152 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-11T18:15:44,152 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,152 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,152 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,154 INFO [Thread-2073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2073. 2024-11-11T18:15:44,154 DEBUG [Thread-2073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,154 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,154 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,155 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-11-11T18:15:44,155 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,155 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,155 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,199 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/33b86f5464284b5494d3a86961f2e173 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/33b86f5464284b5494d3a86961f2e173 2024-11-11T18:15:44,203 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/B of 61e498c0de1e69a7e5cebf09f9286b5a into 33b86f5464284b5494d3a86961f2e173(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:44,203 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:44,203 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/B, priority=13, startTime=1731348943775; duration=0sec 2024-11-11T18:15:44,203 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:44,203 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:B 2024-11-11T18:15:44,220 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/0e993eea304c4ebeacd62036cfdf9bc1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0e993eea304c4ebeacd62036cfdf9bc1 2024-11-11T18:15:44,223 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 61e498c0de1e69a7e5cebf09f9286b5a/C of 61e498c0de1e69a7e5cebf09f9286b5a into 0e993eea304c4ebeacd62036cfdf9bc1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:15:44,223 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:44,223 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., storeName=61e498c0de1e69a7e5cebf09f9286b5a/C, priority=13, startTime=1731348943775; duration=0sec 2024-11-11T18:15:44,223 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:15:44,223 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 61e498c0de1e69a7e5cebf09f9286b5a:C 2024-11-11T18:15:44,288 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/a6c1490abef34c7db585d612591dfbce 2024-11-11T18:15:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/78ad0dc6121548faaf815d3bc161fce5 is 50, key is test_row_0/B:col10/1731348942565/Put/seqid=0 2024-11-11T18:15:44,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742356_1532 (size=12301) 2024-11-11T18:15:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T18:15:44,697 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/78ad0dc6121548faaf815d3bc161fce5 2024-11-11T18:15:44,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7ad8ddac61534fc4ad58f9aa48ba2003 is 50, key is test_row_0/C:col10/1731348942565/Put/seqid=0 2024-11-11T18:15:44,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742357_1533 (size=12301) 2024-11-11T18:15:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. as already flushing 2024-11-11T18:15:44,746 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-11T18:15:44,746 INFO [Thread-2064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2064. 2024-11-11T18:15:44,746 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,746 DEBUG [Thread-2064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,746 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,746 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,746 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,746 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,756 INFO [Thread-2066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2066. 2024-11-11T18:15:44,756 DEBUG [Thread-2066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,756 INFO [Thread-2060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2060. 2024-11-11T18:15:44,756 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,756 DEBUG [Thread-2060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:44,756 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,756 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:44,756 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:44,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:45,106 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7ad8ddac61534fc4ad58f9aa48ba2003 2024-11-11T18:15:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/a6c1490abef34c7db585d612591dfbce as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a6c1490abef34c7db585d612591dfbce 2024-11-11T18:15:45,112 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a6c1490abef34c7db585d612591dfbce, entries=150, sequenceid=309, filesize=12.0 K 2024-11-11T18:15:45,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/78ad0dc6121548faaf815d3bc161fce5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/78ad0dc6121548faaf815d3bc161fce5 2024-11-11T18:15:45,115 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/78ad0dc6121548faaf815d3bc161fce5, entries=150, sequenceid=309, filesize=12.0 K 2024-11-11T18:15:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/7ad8ddac61534fc4ad58f9aa48ba2003 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7ad8ddac61534fc4ad58f9aa48ba2003 2024-11-11T18:15:45,118 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7ad8ddac61534fc4ad58f9aa48ba2003, entries=150, sequenceid=309, filesize=12.0 K 2024-11-11T18:15:45,118 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 61e498c0de1e69a7e5cebf09f9286b5a in 1238ms, sequenceid=309, compaction requested=false 2024-11-11T18:15:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 61e498c0de1e69a7e5cebf09f9286b5a: 2024-11-11T18:15:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-11T18:15:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-11T18:15:45,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T18:15:45,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-11T18:15:45,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.8560 sec 2024-11-11T18:15:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T18:15:45,414 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:48,810 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T18:15:52,676 INFO [Thread-2062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2062. 2024-11-11T18:15:52,676 DEBUG [Thread-2062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:15:52,676 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:52,676 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:52,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:52,676 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1228 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3680 rows 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1220 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3660 rows 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1223 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3661 rows 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1227 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3679 rows 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1229 2024-11-11T18:15:52,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3682 rows 2024-11-11T18:15:52,677 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:15:52,677 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:15:52,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:52,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:52,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:15:52,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T18:15:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T18:15:52,680 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348952680"}]},"ts":"1731348952680"} 2024-11-11T18:15:52,681 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T18:15:52,681 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T18:15:52,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:15:52,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, UNASSIGN}] 2024-11-11T18:15:52,683 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, UNASSIGN 2024-11-11T18:15:52,684 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=61e498c0de1e69a7e5cebf09f9286b5a, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:52,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, UNASSIGN because future has completed 2024-11-11T18:15:52,685 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:15:52,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:15:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T18:15:52,837 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 61e498c0de1e69a7e5cebf09f9286b5a, disabling compactions & flushes 2024-11-11T18:15:52,838 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. after waiting 0 ms 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:52,838 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 61e498c0de1e69a7e5cebf09f9286b5a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=A 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=B 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 61e498c0de1e69a7e5cebf09f9286b5a, store=C 2024-11-11T18:15:52,838 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:52,842 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/60b7d337f4134033b0ebf26cf84297d3 is 50, key is test_row_0/A:col10/1731348944755/Put/seqid=0 2024-11-11T18:15:52,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742358_1534 (size=12301) 2024-11-11T18:15:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T18:15:53,250 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/60b7d337f4134033b0ebf26cf84297d3 2024-11-11T18:15:53,256 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/8e9e1d0786e04fa79babba114dd180cd is 50, key is test_row_0/B:col10/1731348944755/Put/seqid=0 2024-11-11T18:15:53,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742359_1535 (size=12301) 2024-11-11T18:15:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T18:15:53,659 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/8e9e1d0786e04fa79babba114dd180cd 2024-11-11T18:15:53,665 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/882097264ea94367ba8868f8110c164c is 50, key is test_row_0/C:col10/1731348944755/Put/seqid=0 2024-11-11T18:15:53,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742360_1536 (size=12301) 2024-11-11T18:15:53,668 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/882097264ea94367ba8868f8110c164c 2024-11-11T18:15:53,670 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/A/60b7d337f4134033b0ebf26cf84297d3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/60b7d337f4134033b0ebf26cf84297d3 2024-11-11T18:15:53,673 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/60b7d337f4134033b0ebf26cf84297d3, entries=150, sequenceid=319, filesize=12.0 K 2024-11-11T18:15:53,673 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/B/8e9e1d0786e04fa79babba114dd180cd as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/8e9e1d0786e04fa79babba114dd180cd 2024-11-11T18:15:53,676 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/8e9e1d0786e04fa79babba114dd180cd, entries=150, sequenceid=319, filesize=12.0 K 2024-11-11T18:15:53,677 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/.tmp/C/882097264ea94367ba8868f8110c164c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/882097264ea94367ba8868f8110c164c 2024-11-11T18:15:53,679 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/882097264ea94367ba8868f8110c164c, entries=150, sequenceid=319, filesize=12.0 K 2024-11-11T18:15:53,679 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 61e498c0de1e69a7e5cebf09f9286b5a in 841ms, sequenceid=319, compaction requested=true 2024-11-11T18:15:53,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8cd22704734a49b1929b3962c84cc680, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f91c145826404af79743721efc893829, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05c9cd9a39bd44578a2135732102057d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0f1b4b46327e4fbaa9854ae76baa42e1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/e5068ac143554d01b6acd2c4539dc01e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/63d32c75405d4f1e83349c12addb2e92, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8b19cf64d0f14e0b81fd078b0f2595ff, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a3b19a90f95843b8a1230fb1a314b307, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f12a1eb9cd104189895b050374f09b32, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/48c629202bc541f9a472a5d3893fb095, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/ae3e1abaf44b4ff8b052a67e7dbb3618, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/2aa5e949c3614b529fdf6e39e4c2c1e6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/d5f236e4257c4e51b92e8a72efada05f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/358e0d0af2944b96a86edc3529b8446d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/b46fc24c3ca94285ae77809837727a9d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/84e537bae7ef4b22ac61c669d31ac25e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/08a00a87b10b4a70a89069a1b4754045, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/83cae472e392459fb3f4d6c3b1602d12, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/42cb89153f564178a34e53a2bf9f888e, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05be404eed4e430bb766e8a47f89ec5a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/45b1716ed08744ea89485ed91a194480, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0de61a0256d144d8bd82f3be6a6f2f5d] to archive 2024-11-11T18:15:53,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:15:53,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8cd22704734a49b1929b3962c84cc680 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8cd22704734a49b1929b3962c84cc680 2024-11-11T18:15:53,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f91c145826404af79743721efc893829 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f91c145826404af79743721efc893829 2024-11-11T18:15:53,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05c9cd9a39bd44578a2135732102057d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05c9cd9a39bd44578a2135732102057d 2024-11-11T18:15:53,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0f1b4b46327e4fbaa9854ae76baa42e1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0f1b4b46327e4fbaa9854ae76baa42e1 2024-11-11T18:15:53,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/e5068ac143554d01b6acd2c4539dc01e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/e5068ac143554d01b6acd2c4539dc01e 2024-11-11T18:15:53,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/63d32c75405d4f1e83349c12addb2e92 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/63d32c75405d4f1e83349c12addb2e92 2024-11-11T18:15:53,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8b19cf64d0f14e0b81fd078b0f2595ff to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/8b19cf64d0f14e0b81fd078b0f2595ff 2024-11-11T18:15:53,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a3b19a90f95843b8a1230fb1a314b307 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a3b19a90f95843b8a1230fb1a314b307 2024-11-11T18:15:53,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f12a1eb9cd104189895b050374f09b32 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/f12a1eb9cd104189895b050374f09b32 2024-11-11T18:15:53,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/48c629202bc541f9a472a5d3893fb095 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/48c629202bc541f9a472a5d3893fb095 2024-11-11T18:15:53,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/ae3e1abaf44b4ff8b052a67e7dbb3618 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/ae3e1abaf44b4ff8b052a67e7dbb3618 2024-11-11T18:15:53,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/2aa5e949c3614b529fdf6e39e4c2c1e6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/2aa5e949c3614b529fdf6e39e4c2c1e6 2024-11-11T18:15:53,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/d5f236e4257c4e51b92e8a72efada05f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/d5f236e4257c4e51b92e8a72efada05f 2024-11-11T18:15:53,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/358e0d0af2944b96a86edc3529b8446d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/358e0d0af2944b96a86edc3529b8446d 2024-11-11T18:15:53,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/b46fc24c3ca94285ae77809837727a9d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/b46fc24c3ca94285ae77809837727a9d 2024-11-11T18:15:53,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/84e537bae7ef4b22ac61c669d31ac25e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/84e537bae7ef4b22ac61c669d31ac25e 2024-11-11T18:15:53,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/08a00a87b10b4a70a89069a1b4754045 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/08a00a87b10b4a70a89069a1b4754045 2024-11-11T18:15:53,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/83cae472e392459fb3f4d6c3b1602d12 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/83cae472e392459fb3f4d6c3b1602d12 2024-11-11T18:15:53,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/42cb89153f564178a34e53a2bf9f888e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/42cb89153f564178a34e53a2bf9f888e 2024-11-11T18:15:53,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05be404eed4e430bb766e8a47f89ec5a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/05be404eed4e430bb766e8a47f89ec5a 2024-11-11T18:15:53,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/45b1716ed08744ea89485ed91a194480 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/45b1716ed08744ea89485ed91a194480 2024-11-11T18:15:53,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0de61a0256d144d8bd82f3be6a6f2f5d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/0de61a0256d144d8bd82f3be6a6f2f5d 2024-11-11T18:15:53,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/10b04d99ed844118a740d1a751db15cc, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/56df5676bc774e408bf093a80e1ae46a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/166a755e8df045e8a2ce19b65ad216c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/393887301bd7492389bc6e16ba58be77, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/d1cb1027684042abbd596f5d46d54a74, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/12245e340739493889d9cc5105cb2721, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/ac04004ba39d4ac5a046a7107c71466b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/45d1833f79ad4de797210293e40b94a4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/0b3aee9d4d6d4162a039a9b4f983b0de, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/94d1e96a8935446b8dec2b3112193c1f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/274184745e57423fb6234d6b20f8d4ef, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/9aa5175d991d4bd8a02b3d02b5c682a4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/2b66d6574eac4c448e8e5ae1d93b8430, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7120a9dbd89a4c7db4476143983ceccf, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/870385ac6ec24960a5748c8829341427, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/09473bed120d40af83878ca200bc0cba, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/15d0dcaadbd04c11b744e6d6ab842128, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/bae4b477e6d1471f9bd40637f383d48c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7df189bed7fc46fbb2c4592aec8a97bb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/6b2332b6deeb4d10af53d59d3ebc9845, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/b24ff12bd6e44e948d01d980e0f195b1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/a5e1bd57f81941d2bb67a6538f136781] to archive 2024-11-11T18:15:53,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:15:53,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/10b04d99ed844118a740d1a751db15cc to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/10b04d99ed844118a740d1a751db15cc 2024-11-11T18:15:53,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/56df5676bc774e408bf093a80e1ae46a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/56df5676bc774e408bf093a80e1ae46a 2024-11-11T18:15:53,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/166a755e8df045e8a2ce19b65ad216c6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/166a755e8df045e8a2ce19b65ad216c6 2024-11-11T18:15:53,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/393887301bd7492389bc6e16ba58be77 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/393887301bd7492389bc6e16ba58be77 2024-11-11T18:15:53,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/d1cb1027684042abbd596f5d46d54a74 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/d1cb1027684042abbd596f5d46d54a74 2024-11-11T18:15:53,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/12245e340739493889d9cc5105cb2721 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/12245e340739493889d9cc5105cb2721 2024-11-11T18:15:53,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/ac04004ba39d4ac5a046a7107c71466b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/ac04004ba39d4ac5a046a7107c71466b 2024-11-11T18:15:53,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/45d1833f79ad4de797210293e40b94a4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/45d1833f79ad4de797210293e40b94a4 2024-11-11T18:15:53,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/0b3aee9d4d6d4162a039a9b4f983b0de to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/0b3aee9d4d6d4162a039a9b4f983b0de 2024-11-11T18:15:53,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/94d1e96a8935446b8dec2b3112193c1f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/94d1e96a8935446b8dec2b3112193c1f 2024-11-11T18:15:53,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/274184745e57423fb6234d6b20f8d4ef to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/274184745e57423fb6234d6b20f8d4ef 2024-11-11T18:15:53,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/9aa5175d991d4bd8a02b3d02b5c682a4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/9aa5175d991d4bd8a02b3d02b5c682a4 2024-11-11T18:15:53,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/2b66d6574eac4c448e8e5ae1d93b8430 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/2b66d6574eac4c448e8e5ae1d93b8430 2024-11-11T18:15:53,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7120a9dbd89a4c7db4476143983ceccf to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7120a9dbd89a4c7db4476143983ceccf 2024-11-11T18:15:53,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/870385ac6ec24960a5748c8829341427 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/870385ac6ec24960a5748c8829341427 2024-11-11T18:15:53,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/09473bed120d40af83878ca200bc0cba to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/09473bed120d40af83878ca200bc0cba 2024-11-11T18:15:53,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/15d0dcaadbd04c11b744e6d6ab842128 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/15d0dcaadbd04c11b744e6d6ab842128 2024-11-11T18:15:53,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/bae4b477e6d1471f9bd40637f383d48c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/bae4b477e6d1471f9bd40637f383d48c 2024-11-11T18:15:53,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7df189bed7fc46fbb2c4592aec8a97bb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/7df189bed7fc46fbb2c4592aec8a97bb 2024-11-11T18:15:53,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/6b2332b6deeb4d10af53d59d3ebc9845 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/6b2332b6deeb4d10af53d59d3ebc9845 2024-11-11T18:15:53,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/b24ff12bd6e44e948d01d980e0f195b1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/b24ff12bd6e44e948d01d980e0f195b1 2024-11-11T18:15:53,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/a5e1bd57f81941d2bb67a6538f136781 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/a5e1bd57f81941d2bb67a6538f136781 2024-11-11T18:15:53,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44a0c5f35ad04f32b9daa4aa6e61a7e5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44adb52db9df479b871c0a7ca4e3d6d1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/df06f3be54fb4d339bdf082da3312065, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/665ff080293345e3a1d84a661a1cbf95, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/b88cb284319e4284882fc57014966ca9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/5c5b6ba770954bd4893f918dcc074fb5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/865b118d15584babb24aa4fca9ed8ee8, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/345c4fb637724431895ff81fbcfa74d3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/4894b11834ba49e48a96c03cb3dc63da, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/94828aa74b7f4945b5cd7cd1422a3e85, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/abc29a6e124045ac877bae57ea10baaa, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7bd9751ff95e4a6c9dcc1940e71b2760, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/287e530a53e9464898eb67c14ff2626a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/264755dbbece4377bd7f3835f03b9465, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/a05c3bb19c36457fa2ec92c6f771e7ce, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/709119fb3a2f4e75bb9cc5f286677e60, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/486de68295d54e849fd078b21f9eaef9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/c573481f62e144bcb67ffd394c56275a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7c55bc7d0a8c45cf966ffd0a9401697f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/843a296c30cd4760aa7d3b84ef26bd09, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/e78a42c6573c4172b6a5a9918b99069c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0f16dc538a3e4c8e8d10bbf1025c0193] to archive 2024-11-11T18:15:53,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:15:53,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44a0c5f35ad04f32b9daa4aa6e61a7e5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44a0c5f35ad04f32b9daa4aa6e61a7e5 2024-11-11T18:15:53,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44adb52db9df479b871c0a7ca4e3d6d1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/44adb52db9df479b871c0a7ca4e3d6d1 2024-11-11T18:15:53,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/df06f3be54fb4d339bdf082da3312065 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/df06f3be54fb4d339bdf082da3312065 2024-11-11T18:15:53,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/665ff080293345e3a1d84a661a1cbf95 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/665ff080293345e3a1d84a661a1cbf95 2024-11-11T18:15:53,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/b88cb284319e4284882fc57014966ca9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/b88cb284319e4284882fc57014966ca9 2024-11-11T18:15:53,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/5c5b6ba770954bd4893f918dcc074fb5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/5c5b6ba770954bd4893f918dcc074fb5 2024-11-11T18:15:53,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/865b118d15584babb24aa4fca9ed8ee8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/865b118d15584babb24aa4fca9ed8ee8 2024-11-11T18:15:53,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/345c4fb637724431895ff81fbcfa74d3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/345c4fb637724431895ff81fbcfa74d3 2024-11-11T18:15:53,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/4894b11834ba49e48a96c03cb3dc63da to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/4894b11834ba49e48a96c03cb3dc63da 2024-11-11T18:15:53,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/94828aa74b7f4945b5cd7cd1422a3e85 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/94828aa74b7f4945b5cd7cd1422a3e85 2024-11-11T18:15:53,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/abc29a6e124045ac877bae57ea10baaa to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/abc29a6e124045ac877bae57ea10baaa 2024-11-11T18:15:53,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7bd9751ff95e4a6c9dcc1940e71b2760 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7bd9751ff95e4a6c9dcc1940e71b2760 2024-11-11T18:15:53,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/287e530a53e9464898eb67c14ff2626a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/287e530a53e9464898eb67c14ff2626a 2024-11-11T18:15:53,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/264755dbbece4377bd7f3835f03b9465 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/264755dbbece4377bd7f3835f03b9465 2024-11-11T18:15:53,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/a05c3bb19c36457fa2ec92c6f771e7ce to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/a05c3bb19c36457fa2ec92c6f771e7ce 2024-11-11T18:15:53,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/709119fb3a2f4e75bb9cc5f286677e60 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/709119fb3a2f4e75bb9cc5f286677e60 2024-11-11T18:15:53,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/486de68295d54e849fd078b21f9eaef9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/486de68295d54e849fd078b21f9eaef9 2024-11-11T18:15:53,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/c573481f62e144bcb67ffd394c56275a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/c573481f62e144bcb67ffd394c56275a 2024-11-11T18:15:53,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7c55bc7d0a8c45cf966ffd0a9401697f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7c55bc7d0a8c45cf966ffd0a9401697f 2024-11-11T18:15:53,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/843a296c30cd4760aa7d3b84ef26bd09 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/843a296c30cd4760aa7d3b84ef26bd09 2024-11-11T18:15:53,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/e78a42c6573c4172b6a5a9918b99069c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/e78a42c6573c4172b6a5a9918b99069c 2024-11-11T18:15:53,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0f16dc538a3e4c8e8d10bbf1025c0193 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0f16dc538a3e4c8e8d10bbf1025c0193 2024-11-11T18:15:53,737 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/recovered.edits/322.seqid, newMaxSeqId=322, maxSeqId=1 2024-11-11T18:15:53,737 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. 2024-11-11T18:15:53,737 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 61e498c0de1e69a7e5cebf09f9286b5a: Waiting for close lock at 1731348952838Running coprocessor pre-close hooks at 1731348952838Disabling compacts and flushes for region at 1731348952838Disabling writes for close at 1731348952838Obtaining lock to block concurrent updates at 1731348952838Preparing flush snapshotting stores in 61e498c0de1e69a7e5cebf09f9286b5a at 1731348952838Finished memstore snapshotting TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731348952838Flushing stores of TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a. at 1731348952839 (+1 ms)Flushing 61e498c0de1e69a7e5cebf09f9286b5a/A: creating writer at 1731348952839Flushing 61e498c0de1e69a7e5cebf09f9286b5a/A: appending metadata at 1731348952841 (+2 ms)Flushing 61e498c0de1e69a7e5cebf09f9286b5a/A: closing flushed file at 1731348952841Flushing 61e498c0de1e69a7e5cebf09f9286b5a/B: creating writer at 1731348953253 (+412 ms)Flushing 61e498c0de1e69a7e5cebf09f9286b5a/B: appending metadata at 1731348953255 (+2 ms)Flushing 61e498c0de1e69a7e5cebf09f9286b5a/B: closing flushed file at 1731348953255Flushing 61e498c0de1e69a7e5cebf09f9286b5a/C: creating writer at 1731348953662 (+407 ms)Flushing 61e498c0de1e69a7e5cebf09f9286b5a/C: appending metadata at 1731348953664 (+2 ms)Flushing 61e498c0de1e69a7e5cebf09f9286b5a/C: closing flushed file at 1731348953664Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10994b5e: reopening flushed file at 1731348953670 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ad3634a: reopening flushed file at 1731348953673 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46ac03e0: reopening flushed file at 1731348953676 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 61e498c0de1e69a7e5cebf09f9286b5a in 841ms, sequenceid=319, compaction requested=true at 1731348953679 (+3 ms)Writing region close event to WAL at 1731348953735 (+56 ms)Running coprocessor post-close hooks at 1731348953737 (+2 ms)Closed at 1731348953737 2024-11-11T18:15:53,739 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:53,739 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=61e498c0de1e69a7e5cebf09f9286b5a, regionState=CLOSED 2024-11-11T18:15:53,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:15:53,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T18:15:53,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 61e498c0de1e69a7e5cebf09f9286b5a, server=90f34c5bee6a,42957,1731348800579 in 1.0550 sec 2024-11-11T18:15:53,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T18:15:53,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=61e498c0de1e69a7e5cebf09f9286b5a, UNASSIGN in 1.0590 sec 2024-11-11T18:15:53,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T18:15:53,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0620 sec 2024-11-11T18:15:53,746 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348953746"}]},"ts":"1731348953746"} 2024-11-11T18:15:53,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T18:15:53,747 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T18:15:53,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0690 sec 2024-11-11T18:15:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T18:15:53,815 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:53,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T18:15:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T18:15:53,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,819 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:53,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/recovered.edits] 2024-11-11T18:15:53,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/4cd31eab25ab463aa0fc237990bf66a2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/4cd31eab25ab463aa0fc237990bf66a2 2024-11-11T18:15:53,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/60b7d337f4134033b0ebf26cf84297d3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/60b7d337f4134033b0ebf26cf84297d3 2024-11-11T18:15:53,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a6c1490abef34c7db585d612591dfbce to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/A/a6c1490abef34c7db585d612591dfbce 2024-11-11T18:15:53,825 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/33b86f5464284b5494d3a86961f2e173 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/33b86f5464284b5494d3a86961f2e173 2024-11-11T18:15:53,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/78ad0dc6121548faaf815d3bc161fce5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/78ad0dc6121548faaf815d3bc161fce5 2024-11-11T18:15:53,827 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/8e9e1d0786e04fa79babba114dd180cd to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/B/8e9e1d0786e04fa79babba114dd180cd 2024-11-11T18:15:53,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0e993eea304c4ebeacd62036cfdf9bc1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/0e993eea304c4ebeacd62036cfdf9bc1 2024-11-11T18:15:53,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7ad8ddac61534fc4ad58f9aa48ba2003 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/7ad8ddac61534fc4ad58f9aa48ba2003 2024-11-11T18:15:53,830 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/882097264ea94367ba8868f8110c164c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/C/882097264ea94367ba8868f8110c164c 2024-11-11T18:15:53,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/recovered.edits/322.seqid to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a/recovered.edits/322.seqid 2024-11-11T18:15:53,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/61e498c0de1e69a7e5cebf09f9286b5a 2024-11-11T18:15:53,832 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T18:15:53,834 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,835 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T18:15:53,837 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T18:15:53,837 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,837 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T18:15:53,838 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731348953837"}]},"ts":"9223372036854775807"} 2024-11-11T18:15:53,839 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T18:15:53,839 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 61e498c0de1e69a7e5cebf09f9286b5a, NAME => 'TestAcidGuarantees,,1731348921944.61e498c0de1e69a7e5cebf09f9286b5a.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T18:15:53,839 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T18:15:53,839 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731348953839"}]},"ts":"9223372036854775807"} 2024-11-11T18:15:53,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T18:15:53,841 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-11T18:15:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T18:15:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T18:15:53,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:53,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 247), OpenFileDescriptor=443 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 419), ProcessCount=11 (was 11), AvailableMemoryMB=7512 (was 7587) 2024-11-11T18:15:53,943 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=7511 2024-11-11T18:15:53,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:15:53,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:15:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:53,947 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T18:15:53,947 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:53,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-11T18:15:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T18:15:53,947 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T18:15:53,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742361_1537 (size=1061) 2024-11-11T18:15:54,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T18:15:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T18:15:54,353 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7 2024-11-11T18:15:54,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742362_1538 (size=53) 2024-11-11T18:15:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T18:15:54,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:15:54,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fba45d19b0840db7bf05326b8999512f, disabling compactions & flushes 2024-11-11T18:15:54,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:54,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:54,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. after waiting 0 ms 2024-11-11T18:15:54,758 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:54,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:54,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fba45d19b0840db7bf05326b8999512f: Waiting for close lock at 1731348954758Disabling compacts and flushes for region at 1731348954758Disabling writes for close at 1731348954758Writing region close event to WAL at 1731348954758Closed at 1731348954758 2024-11-11T18:15:54,760 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T18:15:54,760 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731348954760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731348954760"}]},"ts":"1731348954760"} 2024-11-11T18:15:54,761 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T18:15:54,762 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T18:15:54,762 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348954762"}]},"ts":"1731348954762"} 2024-11-11T18:15:54,763 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T18:15:54,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, ASSIGN}] 2024-11-11T18:15:54,764 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, ASSIGN 2024-11-11T18:15:54,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, ASSIGN; state=OFFLINE, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=false 2024-11-11T18:15:54,915 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:54,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, ASSIGN because future has completed 2024-11-11T18:15:54,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:15:55,071 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:55,071 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:15:55,071 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,071 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:15:55,071 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,071 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,072 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,073 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:55,074 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fba45d19b0840db7bf05326b8999512f columnFamilyName A 2024-11-11T18:15:55,074 DEBUG [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:55,074 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(327): Store=fba45d19b0840db7bf05326b8999512f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:55,074 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,075 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:55,075 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fba45d19b0840db7bf05326b8999512f columnFamilyName B 2024-11-11T18:15:55,075 DEBUG [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:55,075 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(327): Store=fba45d19b0840db7bf05326b8999512f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:55,075 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,076 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:55,076 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fba45d19b0840db7bf05326b8999512f columnFamilyName C 2024-11-11T18:15:55,076 DEBUG [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:55,076 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(327): Store=fba45d19b0840db7bf05326b8999512f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:55,077 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:55,077 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,077 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,077 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,078 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,078 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,078 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:15:55,079 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,081 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T18:15:55,081 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened fba45d19b0840db7bf05326b8999512f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70624482, jitterRate=0.05238679051399231}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:15:55,081 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:55,082 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for fba45d19b0840db7bf05326b8999512f: Running coprocessor pre-open hook at 1731348955071Writing region info on filesystem at 1731348955071Initializing all the Stores at 1731348955072 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348955072Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348955072Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348955072Cleaning up temporary data from old regions at 1731348955078 (+6 ms)Running coprocessor post-open hooks at 1731348955081 (+3 ms)Region opened successfully at 1731348955082 (+1 ms) 2024-11-11T18:15:55,082 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., pid=148, masterSystemTime=1731348955068 2024-11-11T18:15:55,084 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:55,084 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T18:15:55,084 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=OPEN, openSeqNum=2, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:55,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:15:55,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-11T18:15:55,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 in 169 msec 2024-11-11T18:15:55,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T18:15:55,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, ASSIGN in 324 msec 2024-11-11T18:15:55,089 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T18:15:55,089 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348955089"}]},"ts":"1731348955089"} 2024-11-11T18:15:55,090 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T18:15:55,091 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T18:15:55,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-11T18:15:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T18:15:56,094 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:56,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:56,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:56,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:56,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462906fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:56,098 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c38a16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:56,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:56,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:56,101 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T18:15:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 90f34c5bee6a,41925,1731348799760 2024-11-11T18:15:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f5bb5b2 2024-11-11T18:15:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T18:15:56,102 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T18:15:56,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T18:15:56,104 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T18:15:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T18:15:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742363_1539 (size=1097) 2024-11-11T18:15:56,514 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T18:15:56,514 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T18:15:56,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:15:56,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, REOPEN/MOVE}] 2024-11-11T18:15:56,518 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, REOPEN/MOVE 2024-11-11T18:15:56,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:56,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, REOPEN/MOVE because future has completed 2024-11-11T18:15:56,520 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:15:56,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:15:56,672 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,672 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:15:56,672 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing fba45d19b0840db7bf05326b8999512f, disabling compactions & flushes 2024-11-11T18:15:56,672 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:56,672 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:56,672 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. after waiting 0 ms 2024-11-11T18:15:56,672 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:56,675 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T18:15:56,676 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:56,676 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for fba45d19b0840db7bf05326b8999512f: Waiting for close lock at 1731348956672Running coprocessor pre-close hooks at 1731348956672Disabling compacts and flushes for region at 1731348956672Disabling writes for close at 1731348956672Writing region close event to WAL at 1731348956673 (+1 ms)Running coprocessor post-close hooks at 1731348956676 (+3 ms)Closed at 1731348956676 2024-11-11T18:15:56,676 WARN [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: fba45d19b0840db7bf05326b8999512f to self. 2024-11-11T18:15:56,677 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,678 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=CLOSED 2024-11-11T18:15:56,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:15:56,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-11T18:15:56,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 in 160 msec 2024-11-11T18:15:56,682 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, REOPEN/MOVE; state=CLOSED, location=90f34c5bee6a,42957,1731348800579; forceNewPlan=false, retain=true 2024-11-11T18:15:56,832 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=OPENING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:56,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, REOPEN/MOVE because future has completed 2024-11-11T18:15:56,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:15:56,988 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:56,988 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T18:15:56,988 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,988 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T18:15:56,988 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,988 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,990 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,990 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:56,990 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fba45d19b0840db7bf05326b8999512f columnFamilyName A 2024-11-11T18:15:56,991 DEBUG [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:56,992 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(327): Store=fba45d19b0840db7bf05326b8999512f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:56,992 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,992 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:56,993 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fba45d19b0840db7bf05326b8999512f columnFamilyName B 2024-11-11T18:15:56,993 DEBUG [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:56,993 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(327): Store=fba45d19b0840db7bf05326b8999512f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:56,993 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,993 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T18:15:56,993 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fba45d19b0840db7bf05326b8999512f columnFamilyName C 2024-11-11T18:15:56,994 DEBUG [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:56,994 INFO [StoreOpener-fba45d19b0840db7bf05326b8999512f-1 {}] regionserver.HStore(327): Store=fba45d19b0840db7bf05326b8999512f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T18:15:56,994 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:56,994 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,994 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,995 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,996 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,996 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,996 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T18:15:56,997 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,998 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened fba45d19b0840db7bf05326b8999512f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64887670, jitterRate=-0.03309836983680725}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T18:15:56,998 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:56,998 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for fba45d19b0840db7bf05326b8999512f: Running coprocessor pre-open hook at 1731348956989Writing region info on filesystem at 1731348956989Initializing all the Stores at 1731348956989Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348956989Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348956989Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731348956989Cleaning up temporary data from old regions at 1731348956996 (+7 ms)Running coprocessor post-open hooks at 1731348956998 (+2 ms)Region opened successfully at 1731348956998 2024-11-11T18:15:56,999 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., pid=153, masterSystemTime=1731348956985 2024-11-11T18:15:57,000 DEBUG [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,000 INFO [RS_OPEN_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,001 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=OPEN, openSeqNum=5, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:15:57,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-11T18:15:57,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 in 169 msec 2024-11-11T18:15:57,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T18:15:57,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, REOPEN/MOVE in 487 msec 2024-11-11T18:15:57,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-11T18:15:57,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-11T18:15:57,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-11-11T18:15:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-11T18:15:57,010 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:15:57,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22842932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,011 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62447067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,013 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a378596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,014 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201fb4fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5997ca00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,017 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453a8065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53adf3d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f24d33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ab9b70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,022 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab48957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587cc7a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,025 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648b2e54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66192e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,028 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec06251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,029 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9070b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb4c705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,033 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268b66eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fbcec0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,035 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c47b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 90f34c5bee6a,41925,-1 for getting cluster id 2024-11-11T18:15:57,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T18:15:57,037 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a39067c-aaed-4091-9a8c-8edb05655f25' 2024-11-11T18:15:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T18:15:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a39067c-aaed-4091-9a8c-8edb05655f25" 2024-11-11T18:15:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464b870f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [90f34c5bee6a,41925,-1] 2024-11-11T18:15:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T18:15:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:15:57,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T18:15:57,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5553287, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T18:15:57,039 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,039 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,040 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,040 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,040 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,041 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,042 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:15:57,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,045 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T18:15:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:15:57,047 DEBUG [Thread-2418 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:15:57,047 DEBUG [Thread-2418 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,048 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,048 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T18:15:57,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:15:57,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:15:57,049 DEBUG [Thread-2403 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,049 DEBUG [Thread-2403 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=90f34c5bee6a,42957,1731348800579, seqNum=-1] 2024-11-11T18:15:57,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T18:15:57,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T18:15:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5] 2024-11-11T18:15:57,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:15:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:15:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:15:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:15:57,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:57,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348957074, max timestamp=1731348957076Num uniques 859; 2024-11-11T18:15:57,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348957074, max timestamp=1731348957076Num uniques 859; 2024-11-11T18:15:57,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:15:57,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348957074, max timestamp=1731348957076Num uniques 859; 2024-11-11T18:15:57,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:15:57,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:15:57,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173bb9852f9a240b5b66f13d98bdd5882_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348957051/Put/seqid=0 2024-11-11T18:15:57,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742364_1540 (size=12154) 2024-11-11T18:15:57,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349017096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349017098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349017100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349017101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349017102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:15:57,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349017205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349017214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349017214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349017214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349017214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:57,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:15:57,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349017414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349017424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349017424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349017424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349017424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:57,488 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111173bb9852f9a240b5b66f13d98bdd5882_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173bb9852f9a240b5b66f13d98bdd5882_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:57,489 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/3242613ffb264876af519a95f314a654, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:15:57,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/3242613ffb264876af519a95f314a654 is 175, key is test_row_0/A:col10/1731348957051/Put/seqid=0 2024-11-11T18:15:57,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742365_1541 (size=30955) 2024-11-11T18:15:57,493 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/3242613ffb264876af519a95f314a654 2024-11-11T18:15:57,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/dd3e68f91ca8458f94250e213d702629 is 50, key is test_row_0/B:col10/1731348957051/Put/seqid=0 2024-11-11T18:15:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742366_1542 (size=12001) 2024-11-11T18:15:57,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:15:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349017725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349017734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349017734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349017734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349017735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:57,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:57,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:57,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/dd3e68f91ca8458f94250e213d702629 2024-11-11T18:15:57,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/f4ee6c81abf6444ca4ff5041608ecfa5 is 50, key is test_row_0/C:col10/1731348957051/Put/seqid=0 2024-11-11T18:15:57,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742367_1543 (size=12001) 2024-11-11T18:15:57,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:57,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:58,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:15:58,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349018234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349018244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349018244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349018245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349018245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:58,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:58,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:15:58,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/f4ee6c81abf6444ca4ff5041608ecfa5 2024-11-11T18:15:58,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/3242613ffb264876af519a95f314a654 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654 2024-11-11T18:15:58,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654, entries=150, sequenceid=14, filesize=30.2 K 2024-11-11T18:15:58,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/dd3e68f91ca8458f94250e213d702629 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/dd3e68f91ca8458f94250e213d702629 2024-11-11T18:15:58,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/dd3e68f91ca8458f94250e213d702629, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T18:15:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/f4ee6c81abf6444ca4ff5041608ecfa5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/f4ee6c81abf6444ca4ff5041608ecfa5 2024-11-11T18:15:58,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/f4ee6c81abf6444ca4ff5041608ecfa5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T18:15:58,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fba45d19b0840db7bf05326b8999512f in 1308ms, sequenceid=14, compaction requested=false 2024-11-11T18:15:58,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:15:58,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T18:15:58,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:58,427 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T18:15:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:15:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:15:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:15:58,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:15:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111180000fbcc1d444e1bd0b412c6d7cd33b_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348957084/Put/seqid=0 2024-11-11T18:15:58,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742368_1544 (size=12154) 2024-11-11T18:15:58,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:58,846 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111180000fbcc1d444e1bd0b412c6d7cd33b_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111180000fbcc1d444e1bd0b412c6d7cd33b_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:58,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1d3c4f70b68d4e40adf3503fb88fbc60, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:15:58,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1d3c4f70b68d4e40adf3503fb88fbc60 is 175, key is test_row_0/A:col10/1731348957084/Put/seqid=0 2024-11-11T18:15:58,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742369_1545 (size=30955) 2024-11-11T18:15:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:15:59,252 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1d3c4f70b68d4e40adf3503fb88fbc60 2024-11-11T18:15:59,254 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:59,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:15:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/7f8be4f197584ce8bade7dfa0dd1a44a is 50, key is test_row_0/B:col10/1731348957084/Put/seqid=0 2024-11-11T18:15:59,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349019272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349019273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349019274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742370_1546 (size=12001) 2024-11-11T18:15:59,293 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/7f8be4f197584ce8bade7dfa0dd1a44a 2024-11-11T18:15:59,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349019289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349019290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/9c8e19cb3d82463e8d144f76b30d7b52 is 50, key is test_row_0/C:col10/1731348957084/Put/seqid=0 2024-11-11T18:15:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742371_1547 (size=12001) 2024-11-11T18:15:59,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349019394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349019394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349019394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349019405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349019405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349019604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349019604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349019605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349019614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349019614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,712 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/9c8e19cb3d82463e8d144f76b30d7b52 2024-11-11T18:15:59,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1d3c4f70b68d4e40adf3503fb88fbc60 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60 2024-11-11T18:15:59,719 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60, entries=150, sequenceid=51, filesize=30.2 K 2024-11-11T18:15:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/7f8be4f197584ce8bade7dfa0dd1a44a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/7f8be4f197584ce8bade7dfa0dd1a44a 2024-11-11T18:15:59,723 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/7f8be4f197584ce8bade7dfa0dd1a44a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T18:15:59,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/9c8e19cb3d82463e8d144f76b30d7b52 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/9c8e19cb3d82463e8d144f76b30d7b52 2024-11-11T18:15:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,726 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/9c8e19cb3d82463e8d144f76b30d7b52, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T18:15:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,727 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for fba45d19b0840db7bf05326b8999512f in 1301ms, sequenceid=51, compaction requested=false 2024-11-11T18:15:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:15:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:15:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T18:15:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T18:15:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T18:15:59,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6800 sec 2024-11-11T18:15:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.6870 sec 2024-11-11T18:15:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:15:59,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T18:15:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:15:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:15:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:15:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:15:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111be95aaceaae941928d56a4921b3838b0_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348959917/Put/seqid=0 2024-11-11T18:15:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742372_1548 (size=17034) 2024-11-11T18:15:59,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:15:59,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349019942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349019942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349019943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349019947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:15:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349019947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349020054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349020054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349020054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349020054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349020054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349020264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349020264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349020264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349020265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349020265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,335 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:00,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111be95aaceaae941928d56a4921b3838b0_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be95aaceaae941928d56a4921b3838b0_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:00,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d340d3cdd8ce43a18010bfd9b1897bc4, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:00,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d340d3cdd8ce43a18010bfd9b1897bc4 is 175, key is test_row_0/A:col10/1731348959917/Put/seqid=0 2024-11-11T18:16:00,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742373_1549 (size=48139) 2024-11-11T18:16:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349020574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:00,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d340d3cdd8ce43a18010bfd9b1897bc4 2024-11-11T18:16:00,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/a53bea6f2f04425c99f4b3c1f1b708f5 is 50, key is test_row_0/B:col10/1731348959917/Put/seqid=0 2024-11-11T18:16:00,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742374_1550 (size=12001) 2024-11-11T18:16:00,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/a53bea6f2f04425c99f4b3c1f1b708f5 2024-11-11T18:16:00,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/77e1a593161940d29a4e2eab98de7da3 is 50, key is test_row_0/C:col10/1731348959917/Put/seqid=0 2024-11-11T18:16:00,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742375_1551 (size=12001) 2024-11-11T18:16:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349021084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349021084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:01,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349021084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:01,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349021095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:01,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349021095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:01,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/77e1a593161940d29a4e2eab98de7da3 2024-11-11T18:16:01,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d340d3cdd8ce43a18010bfd9b1897bc4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4 2024-11-11T18:16:01,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4, entries=250, sequenceid=65, filesize=47.0 K 2024-11-11T18:16:01,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/a53bea6f2f04425c99f4b3c1f1b708f5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/a53bea6f2f04425c99f4b3c1f1b708f5 2024-11-11T18:16:01,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/a53bea6f2f04425c99f4b3c1f1b708f5, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T18:16:01,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/77e1a593161940d29a4e2eab98de7da3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/77e1a593161940d29a4e2eab98de7da3 2024-11-11T18:16:01,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/77e1a593161940d29a4e2eab98de7da3, entries=150, sequenceid=65, filesize=11.7 K 2024-11-11T18:16:01,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fba45d19b0840db7bf05326b8999512f in 1267ms, sequenceid=65, compaction requested=true 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:01,184 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:01,184 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:01,185 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:01,185 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:01,185 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:01,185 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:01,185 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:01,185 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:01,185 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=107.5 K 2024-11-11T18:16:01,185 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/dd3e68f91ca8458f94250e213d702629, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/7f8be4f197584ce8bade7dfa0dd1a44a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/a53bea6f2f04425c99f4b3c1f1b708f5] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=35.2 K 2024-11-11T18:16:01,185 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:01,185 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4] 2024-11-11T18:16:01,186 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3e68f91ca8458f94250e213d702629, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731348957051 2024-11-11T18:16:01,186 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3242613ffb264876af519a95f314a654, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731348957051 2024-11-11T18:16:01,186 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3c4f70b68d4e40adf3503fb88fbc60, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731348957084 2024-11-11T18:16:01,186 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8be4f197584ce8bade7dfa0dd1a44a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731348957084 2024-11-11T18:16:01,186 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a53bea6f2f04425c99f4b3c1f1b708f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731348959916 2024-11-11T18:16:01,186 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d340d3cdd8ce43a18010bfd9b1897bc4, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731348959267 2024-11-11T18:16:01,195 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:01,197 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:01,197 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114755f9f4ed8d4f4abf05012abcd54db0_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:01,197 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/18b0b9a8d2bc4e20ac54e7bfef07caf0 is 50, key is test_row_0/B:col10/1731348959917/Put/seqid=0 2024-11-11T18:16:01,199 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114755f9f4ed8d4f4abf05012abcd54db0_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:01,199 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114755f9f4ed8d4f4abf05012abcd54db0_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:01,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742376_1552 (size=12104) 2024-11-11T18:16:01,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742377_1553 (size=4469) 2024-11-11T18:16:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T18:16:01,215 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:01,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:16:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T18:16:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T18:16:01,217 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:16:01,217 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:16:01,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:16:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T18:16:01,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:01,370 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:01,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c07f47322dd74117930e47144ab3a5b5_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348959945/Put/seqid=0 2024-11-11T18:16:01,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742378_1554 (size=12154) 2024-11-11T18:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,385 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c07f47322dd74117930e47144ab3a5b5_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c07f47322dd74117930e47144ab3a5b5_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:01,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f567f9e290de4b91bbb3cc2bc1c75a41, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:01,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f567f9e290de4b91bbb3cc2bc1c75a41 is 175, key is test_row_0/A:col10/1731348959945/Put/seqid=0 2024-11-11T18:16:01,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742379_1555 (size=30955) 2024-11-11T18:16:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T18:16:01,605 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/18b0b9a8d2bc4e20ac54e7bfef07caf0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/18b0b9a8d2bc4e20ac54e7bfef07caf0 2024-11-11T18:16:01,606 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#472 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:01,607 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/5f3ddc9edd694e8485e9e859b75c7962 is 175, key is test_row_0/A:col10/1731348959917/Put/seqid=0 2024-11-11T18:16:01,610 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into 18b0b9a8d2bc4e20ac54e7bfef07caf0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:01,610 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:01,610 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=13, startTime=1731348961184; duration=0sec 2024-11-11T18:16:01,610 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:01,610 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:01,610 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:01,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742380_1556 (size=31058) 2024-11-11T18:16:01,611 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:01,611 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/C is initiating minor compaction (all files) 2024-11-11T18:16:01,611 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/C in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:01,611 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/f4ee6c81abf6444ca4ff5041608ecfa5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/9c8e19cb3d82463e8d144f76b30d7b52, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/77e1a593161940d29a4e2eab98de7da3] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=35.2 K 2024-11-11T18:16:01,612 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting f4ee6c81abf6444ca4ff5041608ecfa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731348957051 2024-11-11T18:16:01,612 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8e19cb3d82463e8d144f76b30d7b52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731348957084 2024-11-11T18:16:01,612 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e1a593161940d29a4e2eab98de7da3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731348959916 2024-11-11T18:16:01,616 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/5f3ddc9edd694e8485e9e859b75c7962 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/5f3ddc9edd694e8485e9e859b75c7962 2024-11-11T18:16:01,621 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into 5f3ddc9edd694e8485e9e859b75c7962(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:01,621 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:01,621 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=13, startTime=1731348961184; duration=0sec 2024-11-11T18:16:01,621 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:01,622 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:01,634 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:01,634 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5160016e83fd475d88f39f5a6c0519f9 is 50, key is test_row_0/C:col10/1731348959917/Put/seqid=0 2024-11-11T18:16:01,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742381_1557 (size=12104) 2024-11-11T18:16:01,792 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f567f9e290de4b91bbb3cc2bc1c75a41 2024-11-11T18:16:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/ca33364ad6e04631b5ae20857003f0b3 is 50, key is test_row_0/B:col10/1731348959945/Put/seqid=0 2024-11-11T18:16:01,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742382_1558 (size=12001) 2024-11-11T18:16:01,803 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/ca33364ad6e04631b5ae20857003f0b3 2024-11-11T18:16:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/3030cf68b2684a97acb20057d9e3ddb5 is 50, key is test_row_0/C:col10/1731348959945/Put/seqid=0 2024-11-11T18:16:01,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742383_1559 (size=12001) 2024-11-11T18:16:01,824 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/3030cf68b2684a97acb20057d9e3ddb5 2024-11-11T18:16:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f567f9e290de4b91bbb3cc2bc1c75a41 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41 2024-11-11T18:16:01,833 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41, entries=150, sequenceid=83, filesize=30.2 K 2024-11-11T18:16:01,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/ca33364ad6e04631b5ae20857003f0b3 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ca33364ad6e04631b5ae20857003f0b3 2024-11-11T18:16:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,839 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ca33364ad6e04631b5ae20857003f0b3, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T18:16:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/3030cf68b2684a97acb20057d9e3ddb5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3030cf68b2684a97acb20057d9e3ddb5 2024-11-11T18:16:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,843 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3030cf68b2684a97acb20057d9e3ddb5, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T18:16:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,844 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for fba45d19b0840db7bf05326b8999512f in 474ms, sequenceid=83, compaction requested=false 2024-11-11T18:16:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T18:16:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T18:16:01,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T18:16:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T18:16:01,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 628 msec 2024-11-11T18:16:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 633 msec 2024-11-11T18:16:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5160016e83fd475d88f39f5a6c0519f9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5160016e83fd475d88f39f5a6c0519f9 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,047 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/C of fba45d19b0840db7bf05326b8999512f into 5160016e83fd475d88f39f5a6c0519f9(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:02,047 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:02,047 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/C, priority=13, startTime=1731348961184; duration=0sec 2024-11-11T18:16:02,047 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:02,047 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:16:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:02,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117dc2e6352d06446eb50378c680c6ed64_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742384_1560 (size=19474) 2024-11-11T18:16:02,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,187 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117dc2e6352d06446eb50378c680c6ed64_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117dc2e6352d06446eb50378c680c6ed64_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:02,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/021435e6f3c04aeb841c932bde4ed204, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:02,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/021435e6f3c04aeb841c932bde4ed204 is 175, key is test_row_0/A:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348962179, max timestamp=1731348962191Num uniques 859; 2024-11-11T18:16:02,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731348962179, max timestamp=1731348962191Num uniques 859; 2024-11-11T18:16:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:16:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T18:16:02,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348962179, max timestamp=1731348962191Num uniques 859; 2024-11-11T18:16:02,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:02,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742385_1561 (size=56733) 2024-11-11T18:16:02,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/021435e6f3c04aeb841c932bde4ed204 2024-11-11T18:16:02,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/24962998409e480086387c8e77e5f628 is 50, key is test_row_0/B:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349022216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349022224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349022224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349022225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349022226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742386_1562 (size=12001) 2024-11-11T18:16:02,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/24962998409e480086387c8e77e5f628 2024-11-11T18:16:02,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a3eebaaaa6b54f59abd32c455bf6c0de is 50, key is test_row_0/C:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742387_1563 (size=12001) 2024-11-11T18:16:02,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a3eebaaaa6b54f59abd32c455bf6c0de 2024-11-11T18:16:02,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/021435e6f3c04aeb841c932bde4ed204 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204 2024-11-11T18:16:02,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204, entries=300, sequenceid=96, filesize=55.4 K 2024-11-11T18:16:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/24962998409e480086387c8e77e5f628 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/24962998409e480086387c8e77e5f628 2024-11-11T18:16:02,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/24962998409e480086387c8e77e5f628, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:16:02,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a3eebaaaa6b54f59abd32c455bf6c0de as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a3eebaaaa6b54f59abd32c455bf6c0de 2024-11-11T18:16:02,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a3eebaaaa6b54f59abd32c455bf6c0de, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T18:16:02,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fba45d19b0840db7bf05326b8999512f in 136ms, sequenceid=96, compaction requested=true 2024-11-11T18:16:02,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:02,288 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:02,289 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:02,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:02,291 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:02,291 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:02,291 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,291 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/18b0b9a8d2bc4e20ac54e7bfef07caf0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ca33364ad6e04631b5ae20857003f0b3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/24962998409e480086387c8e77e5f628] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=35.3 K 2024-11-11T18:16:02,291 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:02,291 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:02,291 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,291 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/5f3ddc9edd694e8485e9e859b75c7962, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=116.0 K 2024-11-11T18:16:02,291 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,291 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/5f3ddc9edd694e8485e9e859b75c7962, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204] 2024-11-11T18:16:02,292 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f3ddc9edd694e8485e9e859b75c7962, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731348959916 2024-11-11T18:16:02,292 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b0b9a8d2bc4e20ac54e7bfef07caf0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731348959916 2024-11-11T18:16:02,292 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting ca33364ad6e04631b5ae20857003f0b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731348959940 2024-11-11T18:16:02,292 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f567f9e290de4b91bbb3cc2bc1c75a41, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731348959940 2024-11-11T18:16:02,293 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 021435e6f3c04aeb841c932bde4ed204, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348962125 2024-11-11T18:16:02,293 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 24962998409e480086387c8e77e5f628, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348962138 2024-11-11T18:16:02,300 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:02,313 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:02,314 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/1cd2590d3c3443eabaf2c99589565f85 is 50, key is test_row_0/B:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,316 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111260311ea4fce437aba84606c3de70b85_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:02,318 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111260311ea4fce437aba84606c3de70b85_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:02,318 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111260311ea4fce437aba84606c3de70b85_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:02,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742388_1564 (size=12207) 2024-11-11T18:16:02,338 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/1cd2590d3c3443eabaf2c99589565f85 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1cd2590d3c3443eabaf2c99589565f85 2024-11-11T18:16:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:02,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-11T18:16:02,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742389_1565 (size=4469) 2024-11-11T18:16:02,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:02,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:02,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:02,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:02,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:02,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:02,354 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into 1cd2590d3c3443eabaf2c99589565f85(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:02,354 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:02,354 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=13, startTime=1731348962289; duration=0sec 2024-11-11T18:16:02,354 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:02,354 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:02,354 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T18:16:02,355 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:02,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:16:02,357 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:02,357 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/C is initiating minor compaction (all files) 2024-11-11T18:16:02,357 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/C in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,358 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5160016e83fd475d88f39f5a6c0519f9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3030cf68b2684a97acb20057d9e3ddb5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a3eebaaaa6b54f59abd32c455bf6c0de] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=35.3 K 2024-11-11T18:16:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-11T18:16:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:02,359 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:16:02,359 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5160016e83fd475d88f39f5a6c0519f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731348959916 2024-11-11T18:16:02,360 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:16:02,360 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3030cf68b2684a97acb20057d9e3ddb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731348959940 2024-11-11T18:16:02,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:16:02,361 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a3eebaaaa6b54f59abd32c455bf6c0de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348962138 2024-11-11T18:16:02,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d77df6898af44dd0abf775839a1ce128_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348962221/Put/seqid=0 2024-11-11T18:16:02,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349022353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349022353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349022357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349022359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349022360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742390_1566 (size=12254) 2024-11-11T18:16:02,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:02,373 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#C#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:02,374 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/d8d8d8bda5e747fca573d736a0342cf1 is 50, key is test_row_0/C:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d77df6898af44dd0abf775839a1ce128_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d77df6898af44dd0abf775839a1ce128_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:02,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b3580e1748da4db2884d0c10dacf6ec5, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:02,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b3580e1748da4db2884d0c10dacf6ec5 is 175, key is test_row_0/A:col10/1731348962221/Put/seqid=0 2024-11-11T18:16:02,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742391_1567 (size=12207) 2024-11-11T18:16:02,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742392_1568 (size=31055) 2024-11-11T18:16:02,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b3580e1748da4db2884d0c10dacf6ec5 2024-11-11T18:16:02,402 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/d8d8d8bda5e747fca573d736a0342cf1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d8d8d8bda5e747fca573d736a0342cf1 2024-11-11T18:16:02,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/3e99180330f0416db68663d3a138f23a is 50, key is test_row_0/B:col10/1731348962221/Put/seqid=0 2024-11-11T18:16:02,411 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/C of fba45d19b0840db7bf05326b8999512f into d8d8d8bda5e747fca573d736a0342cf1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:02,411 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:02,411 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/C, priority=13, startTime=1731348962289; duration=0sec 2024-11-11T18:16:02,411 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:02,411 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:02,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742393_1569 (size=12101) 2024-11-11T18:16:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:02,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349022474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349022474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349022474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349022475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349022475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T18:16:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T18:16:02,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:02,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:02,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349022684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349022684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349022685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349022685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349022685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,743 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#480 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:02,744 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/88dd2152ce9f498c9f70f6fa75b5d131 is 175, key is test_row_0/A:col10/1731348962149/Put/seqid=0 2024-11-11T18:16:02,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742394_1570 (size=31161) 2024-11-11T18:16:02,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T18:16:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/3e99180330f0416db68663d3a138f23a 2024-11-11T18:16:02,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/b93c8a2e3f1f4cbdb50544aac3ca4233 is 50, key is test_row_0/C:col10/1731348962221/Put/seqid=0 2024-11-11T18:16:02,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742395_1571 (size=12101) 2024-11-11T18:16:02,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T18:16:02,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:02,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:02,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:02,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349022994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349022994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349022995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349022995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349022995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T18:16:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:03,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:03,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:03,151 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/88dd2152ce9f498c9f70f6fa75b5d131 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/88dd2152ce9f498c9f70f6fa75b5d131 2024-11-11T18:16:03,155 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into 88dd2152ce9f498c9f70f6fa75b5d131(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:03,155 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:03,155 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=13, startTime=1731348962288; duration=0sec 2024-11-11T18:16:03,155 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:03,155 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:03,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/b93c8a2e3f1f4cbdb50544aac3ca4233 2024-11-11T18:16:03,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b3580e1748da4db2884d0c10dacf6ec5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5 2024-11-11T18:16:03,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5, entries=150, sequenceid=133, filesize=30.3 K 2024-11-11T18:16:03,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/3e99180330f0416db68663d3a138f23a as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3e99180330f0416db68663d3a138f23a 2024-11-11T18:16:03,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3e99180330f0416db68663d3a138f23a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T18:16:03,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/b93c8a2e3f1f4cbdb50544aac3ca4233 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b93c8a2e3f1f4cbdb50544aac3ca4233 2024-11-11T18:16:03,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b93c8a2e3f1f4cbdb50544aac3ca4233, entries=150, sequenceid=133, filesize=11.8 K 2024-11-11T18:16:03,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for fba45d19b0840db7bf05326b8999512f in 914ms, sequenceid=133, compaction requested=false 2024-11-11T18:16:03,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:03,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T18:16:03,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:03,278 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T18:16:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:03,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:03,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a82f831cfa24101b1bd6bb525a0c861_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348962356/Put/seqid=0 2024-11-11T18:16:03,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742396_1572 (size=12304) 2024-11-11T18:16:03,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:03,291 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a82f831cfa24101b1bd6bb525a0c861_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a82f831cfa24101b1bd6bb525a0c861_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:03,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f70d7ebb3c2245de8764031b89a262c6, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:03,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f70d7ebb3c2245de8764031b89a262c6 is 175, key is test_row_0/A:col10/1731348962356/Put/seqid=0 2024-11-11T18:16:03,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742397_1573 (size=31105) 2024-11-11T18:16:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:03,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:03,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348963521, max timestamp=1731348963527Num uniques 859; 2024-11-11T18:16:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:16:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348963521, max timestamp=1731348963527Num uniques 859; 2024-11-11T18:16:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348963521, max timestamp=1731348963527Num uniques 859; 2024-11-11T18:16:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:16:03,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:16:03,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349023548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349023550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349023551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349023552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349023553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349023665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349023665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349023665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349023665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349023665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,698 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f70d7ebb3c2245de8764031b89a262c6 2024-11-11T18:16:03,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/908a80781d4b4acebf873abaab6cd527 is 50, key is test_row_0/B:col10/1731348962356/Put/seqid=0 2024-11-11T18:16:03,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742398_1574 (size=12151) 2024-11-11T18:16:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349023875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349023875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349023875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349023875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349023875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,109 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/908a80781d4b4acebf873abaab6cd527 2024-11-11T18:16:04,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5cf354d114dd4fe18c9cbcfe37eb75e4 is 50, key is test_row_0/C:col10/1731348962356/Put/seqid=0 2024-11-11T18:16:04,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742399_1575 (size=12151) 2024-11-11T18:16:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349024185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349024185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349024185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349024185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349024185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:04,522 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5cf354d114dd4fe18c9cbcfe37eb75e4 2024-11-11T18:16:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f70d7ebb3c2245de8764031b89a262c6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6 2024-11-11T18:16:04,529 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6, entries=150, sequenceid=145, filesize=30.4 K 2024-11-11T18:16:04,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/908a80781d4b4acebf873abaab6cd527 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/908a80781d4b4acebf873abaab6cd527 2024-11-11T18:16:04,533 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/908a80781d4b4acebf873abaab6cd527, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T18:16:04,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5cf354d114dd4fe18c9cbcfe37eb75e4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5cf354d114dd4fe18c9cbcfe37eb75e4 2024-11-11T18:16:04,537 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5cf354d114dd4fe18c9cbcfe37eb75e4, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T18:16:04,544 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for fba45d19b0840db7bf05326b8999512f in 1266ms, sequenceid=145, compaction requested=true 2024-11-11T18:16:04,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:04,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:04,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-11T18:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-11T18:16:04,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T18:16:04,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1860 sec 2024-11-11T18:16:04,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1920 sec 2024-11-11T18:16:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:04,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-11T18:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:04,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:16:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349024703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349024703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349024703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118b9238cd514044f08a648a91cdcb2f8d_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742400_1576 (size=14794) 2024-11-11T18:16:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349024707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349024707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349024814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349024814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349024814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349024825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349024825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349025024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349025024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349025024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349025035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349025035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,116 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118b9238cd514044f08a648a91cdcb2f8d_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b9238cd514044f08a648a91cdcb2f8d_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:05,117 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/41980c1652a048319f1e7da2acd0156c, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:05,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/41980c1652a048319f1e7da2acd0156c is 175, key is test_row_0/A:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:05,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742401_1577 (size=39749) 2024-11-11T18:16:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349025334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349025334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349025334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349025344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349025345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,521 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/41980c1652a048319f1e7da2acd0156c 2024-11-11T18:16:05,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/3276bc6ba42a432daa07f43901c5560d is 50, key is test_row_0/B:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:05,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742402_1578 (size=12151) 2024-11-11T18:16:05,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/3276bc6ba42a432daa07f43901c5560d 2024-11-11T18:16:05,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5975f2b6cb4b4623a5677730ddafbe1b is 50, key is test_row_0/C:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:05,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742403_1579 (size=12151) 2024-11-11T18:16:05,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349025844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349025844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349025844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349025855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349025864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:05,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:05,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:05,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5975f2b6cb4b4623a5677730ddafbe1b 2024-11-11T18:16:05,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/41980c1652a048319f1e7da2acd0156c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c 2024-11-11T18:16:05,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c, entries=200, sequenceid=186, filesize=38.8 K 2024-11-11T18:16:05,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/3276bc6ba42a432daa07f43901c5560d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3276bc6ba42a432daa07f43901c5560d 2024-11-11T18:16:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3276bc6ba42a432daa07f43901c5560d, entries=150, sequenceid=186, filesize=11.9 K 2024-11-11T18:16:05,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/5975f2b6cb4b4623a5677730ddafbe1b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5975f2b6cb4b4623a5677730ddafbe1b 2024-11-11T18:16:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5975f2b6cb4b4623a5677730ddafbe1b, entries=150, sequenceid=186, filesize=11.9 K 2024-11-11T18:16:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~480.56 KB/492096, currentSize=20.13 KB/20610 for fba45d19b0840db7bf05326b8999512f in 1261ms, sequenceid=186, compaction requested=true 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:05,961 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,961 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:16:05,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,962 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:16:05,962 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133070 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:16:05,963 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:05,963 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:05,963 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:05,963 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1cd2590d3c3443eabaf2c99589565f85, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3e99180330f0416db68663d3a138f23a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/908a80781d4b4acebf873abaab6cd527, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3276bc6ba42a432daa07f43901c5560d] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=47.5 K 2024-11-11T18:16:05,963 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:05,963 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/88dd2152ce9f498c9f70f6fa75b5d131, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=130.0 K 2024-11-11T18:16:05,963 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:05,963 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/88dd2152ce9f498c9f70f6fa75b5d131, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c] 2024-11-11T18:16:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,963 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd2590d3c3443eabaf2c99589565f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348962138 2024-11-11T18:16:05,963 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88dd2152ce9f498c9f70f6fa75b5d131, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348962138 2024-11-11T18:16:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,964 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e99180330f0416db68663d3a138f23a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348962221 2024-11-11T18:16:05,964 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3580e1748da4db2884d0c10dacf6ec5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348962221 2024-11-11T18:16:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,965 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 908a80781d4b4acebf873abaab6cd527, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731348962347 2024-11-11T18:16:05,965 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70d7ebb3c2245de8764031b89a262c6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731348962347 2024-11-11T18:16:05,965 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 3276bc6ba42a432daa07f43901c5560d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731348963546 2024-11-11T18:16:05,965 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41980c1652a048319f1e7da2acd0156c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731348963546 2024-11-11T18:16:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,978 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:05,979 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/8ebe0de73a7b4018a213f71f34eb0d7f is 50, key is test_row_0/B:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,989 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742404_1580 (size=12493) 2024-11-11T18:16:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,001 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411115c55c1e10c514afd8b79536aa04e2bc7_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,003 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411115c55c1e10c514afd8b79536aa04e2bc7_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:06,004 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c55c1e10c514afd8b79536aa04e2bc7_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742405_1581 (size=4469) 2024-11-11T18:16:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,046 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#493 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,046 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1cde7e3948d54c1f9a306885bbaad45d is 175, key is test_row_0/A:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742406_1582 (size=31447) 2024-11-11T18:16:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,056 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1cde7e3948d54c1f9a306885bbaad45d as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1cde7e3948d54c1f9a306885bbaad45d 2024-11-11T18:16:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,062 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into 1cde7e3948d54c1f9a306885bbaad45d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:06,062 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:06,062 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=12, startTime=1731348965961; duration=0sec 2024-11-11T18:16:06,062 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:06,062 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:06,062 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:16:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,063 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:16:06,063 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/C is initiating minor compaction (all files) 2024-11-11T18:16:06,064 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/C in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:06,064 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d8d8d8bda5e747fca573d736a0342cf1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b93c8a2e3f1f4cbdb50544aac3ca4233, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5cf354d114dd4fe18c9cbcfe37eb75e4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5975f2b6cb4b4623a5677730ddafbe1b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=47.5 K 2024-11-11T18:16:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,065 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d8d8bda5e747fca573d736a0342cf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731348962138 2024-11-11T18:16:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,065 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93c8a2e3f1f4cbdb50544aac3ca4233, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731348962221 2024-11-11T18:16:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,065 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf354d114dd4fe18c9cbcfe37eb75e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731348962347 2024-11-11T18:16:06,066 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5975f2b6cb4b4623a5677730ddafbe1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731348963546 2024-11-11T18:16:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,094 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,095 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/d7d80c86a2664b34902f61da21f98b10 is 50, key is test_row_0/C:col10/1731348964699/Put/seqid=0 2024-11-11T18:16:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742407_1583 (size=12493) 2024-11-11T18:16:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,117 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/d7d80c86a2664b34902f61da21f98b10 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d7d80c86a2664b34902f61da21f98b10 2024-11-11T18:16:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,121 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fba45d19b0840db7bf05326b8999512f/C of fba45d19b0840db7bf05326b8999512f into d7d80c86a2664b34902f61da21f98b10(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:06,121 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:06,121 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/C, priority=12, startTime=1731348965961; duration=0sec 2024-11-11T18:16:06,122 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:06,122 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,409 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/8ebe0de73a7b4018a213f71f34eb0d7f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/8ebe0de73a7b4018a213f71f34eb0d7f 2024-11-11T18:16:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,414 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into 8ebe0de73a7b4018a213f71f34eb0d7f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:06,414 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:06,414 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=12, startTime=1731348965961; duration=0sec 2024-11-11T18:16:06,414 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:06,414 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T18:16:06,524 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:06,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-11T18:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T18:16:06,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,527 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:16:06,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:16:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T18:16:06,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T18:16:06,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:06,679 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T18:16:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:06,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e44195864aa64f3f88206f26aeea73a2_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348964706/Put/seqid=0 2024-11-11T18:16:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742408_1584 (size=9814) 2024-11-11T18:16:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T18:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:06,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:06,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348966878, max timestamp=1731348966886Num uniques 859; 2024-11-11T18:16:06,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:16:06,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731348966878, max timestamp=1731348966886Num uniques 859; 2024-11-11T18:16:06,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=152, cellSize=6960, totalHeapSize=16048, min timestamp=1731348966878, max timestamp=1731348966887Num uniques 859; 2024-11-11T18:16:06,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:16:06,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39332, new segment data size=6960, suffix heap size=117064,new segment heap size=16048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=152 2024-11-11T18:16:06,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731348966905, max timestamp=1731348966913Num uniques 1011; 2024-11-11T18:16:06,918 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46306, new segment data size=9978, suffix heap size=133112,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1011, new segment cells count=218 2024-11-11T18:16:06,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9974, totalHeapSize=22912, min timestamp=1731348966905, max timestamp=1731348966913Num uniques 1068; 2024-11-11T18:16:06,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48904, new segment data size=9974, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T18:16:06,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9971, totalHeapSize=22912, min timestamp=1731348966905, max timestamp=1731348966913Num uniques 1068; 2024-11-11T18:16:06,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48901, new segment data size=9971, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T18:16:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349026937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:06,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731348966933, max timestamp=1731348966940Num uniques 1077; 2024-11-11T18:16:06,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731348966933, max timestamp=1731348966940Num uniques 1077; 2024-11-11T18:16:06,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T18:16:06,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T18:16:06,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731348966933, max timestamp=1731348966940Num uniques 1077; 2024-11-11T18:16:06,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-11T18:16:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:06,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731348966958, max timestamp=1731348966970Num uniques 1086; 2024-11-11T18:16:06,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-11T18:16:06,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731348966958, max timestamp=1731348966970Num uniques 1086; 2024-11-11T18:16:06,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-11T18:16:06,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731348966958, max timestamp=1731348966970Num uniques 1086; 2024-11-11T18:16:06,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-11T18:16:07,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731348967012, max timestamp=1731348967026Num uniques 1145; 2024-11-11T18:16:07,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-11T18:16:07,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731348967012, max timestamp=1731348967026Num uniques 1145; 2024-11-11T18:16:07,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-11T18:16:07,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731348967012, max timestamp=1731348967026Num uniques 1145; 2024-11-11T18:16:07,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-11T18:16:07,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349027053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349027054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731348967045, max timestamp=1731348967056Num uniques 1154; 2024-11-11T18:16:07,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-11T18:16:07,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731348967045, max timestamp=1731348967056Num uniques 1154; 2024-11-11T18:16:07,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-11T18:16:07,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731348967045, max timestamp=1731348967056Num uniques 1154; 2024-11-11T18:16:07,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-11T18:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731348967069, max timestamp=1731348967090Num uniques 1063; 2024-11-11T18:16:07,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-11T18:16:07,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731348967069, max timestamp=1731348967090Num uniques 1063; 2024-11-11T18:16:07,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731348967069, max timestamp=1731348967090Num uniques 1063; 2024-11-11T18:16:07,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-11T18:16:07,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-11T18:16:07,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:07,105 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e44195864aa64f3f88206f26aeea73a2_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e44195864aa64f3f88206f26aeea73a2_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:07,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d5f4bce6d0064dcabc3ba740ac3b0019, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:07,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d5f4bce6d0064dcabc3ba740ac3b0019 is 175, key is test_row_0/A:col10/1731348964706/Put/seqid=0 2024-11-11T18:16:07,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742409_1585 (size=22461) 2024-11-11T18:16:07,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731348967112, max timestamp=1731348967130Num uniques 1072; 2024-11-11T18:16:07,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731348967112, max timestamp=1731348967130Num uniques 1072; 2024-11-11T18:16:07,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T18:16:07,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731348967112, max timestamp=1731348967130Num uniques 1072; 2024-11-11T18:16:07,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T18:16:07,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-11T18:16:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T18:16:07,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731348967151, max timestamp=1731348967165Num uniques 1081; 2024-11-11T18:16:07,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T18:16:07,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731348967151, max timestamp=1731348967165Num uniques 1081; 2024-11-11T18:16:07,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T18:16:07,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731348967151, max timestamp=1731348967165Num uniques 1081; 2024-11-11T18:16:07,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-11T18:16:07,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1731348967168, max timestamp=1731348967197Num uniques 1090; 2024-11-11T18:16:07,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=8690, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-11T18:16:07,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1731348967168, max timestamp=1731348967197Num uniques 1090; 2024-11-11T18:16:07,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8693, totalHeapSize=20000, min timestamp=1731348967168, max timestamp=1731348967197Num uniques 1090; 2024-11-11T18:16:07,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49910, new segment data size=8690, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-11T18:16:07,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49913, new segment data size=8693, suffix heap size=141328,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=190 2024-11-11T18:16:07,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731348967218, max timestamp=1731348967236Num uniques 1049; 2024-11-11T18:16:07,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731348967218, max timestamp=1731348967236Num uniques 1049; 2024-11-11T18:16:07,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-11T18:16:07,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-11T18:16:07,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731348967218, max timestamp=1731348967236Num uniques 1049; 2024-11-11T18:16:07,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-11T18:16:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731348967260, max timestamp=1731348967276Num uniques 1108; 2024-11-11T18:16:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731348967260, max timestamp=1731348967276Num uniques 1108; 2024-11-11T18:16:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-11T18:16:07,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-11T18:16:07,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731348967260, max timestamp=1731348967276Num uniques 1108; 2024-11-11T18:16:07,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-11T18:16:07,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731348967298, max timestamp=1731348967308Num uniques 1067; 2024-11-11T18:16:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-11T18:16:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7639, totalHeapSize=17608, min timestamp=1731348967298, max timestamp=1731348967309Num uniques 1067; 2024-11-11T18:16:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48859, new segment data size=7639, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-11T18:16:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7639, totalHeapSize=17608, min timestamp=1731348967298, max timestamp=1731348967309Num uniques 1067; 2024-11-11T18:16:07,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48859, new segment data size=7639, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-11T18:16:07,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731348967332, max timestamp=1731348967341Num uniques 1026; 2024-11-11T18:16:07,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731348967332, max timestamp=1731348967341Num uniques 1026; 2024-11-11T18:16:07,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=12636, suffix heap size=134672,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=276 2024-11-11T18:16:07,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=12636, suffix heap size=134672,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=276 2024-11-11T18:16:07,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=276, cellSize=12636, totalHeapSize=28944, min timestamp=1731348967332, max timestamp=1731348967341Num uniques 1026; 2024-11-11T18:16:07,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=12636, suffix heap size=134672,new segment heap size=28944 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=276 2024-11-11T18:16:07,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731348967351, max timestamp=1731348967372Num uniques 1135; 2024-11-11T18:16:07,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=13050, suffix heap size=146008,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=285 2024-11-11T18:16:07,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731348967351, max timestamp=1731348967372Num uniques 1135; 2024-11-11T18:16:07,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731348967351, max timestamp=1731348967372Num uniques 1135; 2024-11-11T18:16:07,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=13050, suffix heap size=146008,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=285 2024-11-11T18:16:07,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51980, new segment data size=13050, suffix heap size=146008,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1135, new segment cells count=285 2024-11-11T18:16:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349027395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349027397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731348967394, max timestamp=1731348967399Num uniques 1144; 2024-11-11T18:16:07,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-11T18:16:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731348967394, max timestamp=1731348967399Num uniques 1144; 2024-11-11T18:16:07,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-11T18:16:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349027396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8884, totalHeapSize=20416, min timestamp=1731348967394, max timestamp=1731348967399Num uniques 1144; 2024-11-11T18:16:07,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=8884, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-11T18:16:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731348967435, max timestamp=1731348967447Num uniques 1053; 2024-11-11T18:16:07,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-11T18:16:07,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731348967435, max timestamp=1731348967447Num uniques 1053; 2024-11-11T18:16:07,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-11T18:16:07,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=153, cellSize=7005, totalHeapSize=16152, min timestamp=1731348967435, max timestamp=1731348967447Num uniques 1053; 2024-11-11T18:16:07,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=7005, suffix heap size=137480,new segment heap size=16152 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=153 2024-11-11T18:16:07,511 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d5f4bce6d0064dcabc3ba740ac3b0019 2024-11-11T18:16:07,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731348967500, max timestamp=1731348967515Num uniques 1012; 2024-11-11T18:16:07,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731348967500, max timestamp=1731348967515Num uniques 1012; 2024-11-11T18:16:07,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-11T18:16:07,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-11T18:16:07,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731348967500, max timestamp=1731348967515Num uniques 1012; 2024-11-11T18:16:07,520 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46342, new segment data size=9702, suffix heap size=133216,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1012, new segment cells count=212 2024-11-11T18:16:07,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/40a8e996c9cd4bc1bbded829bf4dbde6 is 50, key is test_row_0/B:col10/1731348964706/Put/seqid=0 2024-11-11T18:16:07,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742410_1586 (size=9757) 2024-11-11T18:16:07,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349027536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731348967531, max timestamp=1731348967538Num uniques 1071; 2024-11-11T18:16:07,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-11T18:16:07,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731348967531, max timestamp=1731348967538Num uniques 1071; 2024-11-11T18:16:07,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-11T18:16:07,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731348967531, max timestamp=1731348967538Num uniques 1071; 2024-11-11T18:16:07,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-11T18:16:07,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1731348967562, max timestamp=1731348967571Num uniques 1080; 2024-11-11T18:16:07,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=12820, suffix heap size=140288,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=280 2024-11-11T18:16:07,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1731348967562, max timestamp=1731348967571Num uniques 1080; 2024-11-11T18:16:07,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=280, cellSize=12820, totalHeapSize=29360, min timestamp=1731348967562, max timestamp=1731348967571Num uniques 1080; 2024-11-11T18:16:07,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=12820, suffix heap size=140288,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=280 2024-11-11T18:16:07,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=12820, suffix heap size=140288,new segment heap size=29360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=280 2024-11-11T18:16:07,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1731348967589, max timestamp=1731348967599Num uniques 1139; 2024-11-11T18:16:07,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=13234, suffix heap size=146424,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=289 2024-11-11T18:16:07,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1731348967589, max timestamp=1731348967599Num uniques 1139; 2024-11-11T18:16:07,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=289, cellSize=13234, totalHeapSize=30296, min timestamp=1731348967589, max timestamp=1731348967599Num uniques 1139; 2024-11-11T18:16:07,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=13234, suffix heap size=146424,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=289 2024-11-11T18:16:07,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52164, new segment data size=13234, suffix heap size=146424,new segment heap size=30296 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1139, new segment cells count=289 2024-11-11T18:16:07,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1731348967617, max timestamp=1731348967626Num uniques 1148; 2024-11-11T18:16:07,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-11T18:16:07,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1731348967617, max timestamp=1731348967626Num uniques 1148; 2024-11-11T18:16:07,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-11T18:16:07,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=298, cellSize=13648, totalHeapSize=31232, min timestamp=1731348967617, max timestamp=1731348967626Num uniques 1148; 2024-11-11T18:16:07,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52578, new segment data size=13648, suffix heap size=147360,new segment heap size=31232 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1148, new segment cells count=298 2024-11-11T18:16:07,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349027654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349027655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349027659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731348967652, max timestamp=1731348967659Num uniques 1157; 2024-11-11T18:16:07,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=9475, suffix heap size=148296,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=207 2024-11-11T18:16:07,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731348967652, max timestamp=1731348967659Num uniques 1157; 2024-11-11T18:16:07,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=207, cellSize=9475, totalHeapSize=21768, min timestamp=1731348967652, max timestamp=1731348967659Num uniques 1157; 2024-11-11T18:16:07,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=9475, suffix heap size=148296,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=207 2024-11-11T18:16:07,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52985, new segment data size=9475, suffix heap size=148296,new segment heap size=21768 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1157, new segment cells count=207 2024-11-11T18:16:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T18:16:07,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731348967687, max timestamp=1731348967705Num uniques 1066; 2024-11-11T18:16:07,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731348967687, max timestamp=1731348967705Num uniques 1066; 2024-11-11T18:16:07,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-11T18:16:07,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-11T18:16:07,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1731348967687, max timestamp=1731348967705Num uniques 1066; 2024-11-11T18:16:07,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48816, new segment data size=7596, suffix heap size=138832,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1066, new segment cells count=166 2024-11-11T18:16:07,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349027774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731348967769, max timestamp=1731348967778Num uniques 1025; 2024-11-11T18:16:07,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731348967769, max timestamp=1731348967778Num uniques 1025; 2024-11-11T18:16:07,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=175, cellSize=8010, totalHeapSize=18440, min timestamp=1731348967769, max timestamp=1731348967778Num uniques 1025; 2024-11-11T18:16:07,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-11T18:16:07,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-11T18:16:07,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46940, new segment data size=8010, suffix heap size=134568,new segment heap size=18440 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1025, new segment cells count=175 2024-11-11T18:16:07,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1731348967797, max timestamp=1731348967808Num uniques 1034; 2024-11-11T18:16:07,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=10714, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-11-11T18:16:07,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1731348967797, max timestamp=1731348967808Num uniques 1034; 2024-11-11T18:16:07,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=234, cellSize=10714, totalHeapSize=24576, min timestamp=1731348967797, max timestamp=1731348967808Num uniques 1034; 2024-11-11T18:16:07,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=10714, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-11-11T18:16:07,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47354, new segment data size=10714, suffix heap size=135504,new segment heap size=24576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1034, new segment cells count=234 2024-11-11T18:16:07,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8838, totalHeapSize=20312, min timestamp=1731348967831, max timestamp=1731348967841Num uniques 1093; 2024-11-11T18:16:07,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50058, new segment data size=8838, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-11-11T18:16:07,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8832, totalHeapSize=20312, min timestamp=1731348967831, max timestamp=1731348967841Num uniques 1093; 2024-11-11T18:16:07,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50052, new segment data size=8832, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-11-11T18:16:07,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=193, cellSize=8831, totalHeapSize=20312, min timestamp=1731348967831, max timestamp=1731348967841Num uniques 1093; 2024-11-11T18:16:07,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50051, new segment data size=8831, suffix heap size=141640,new segment heap size=20312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1093, new segment cells count=193 2024-11-11T18:16:07,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=202, cellSize=9250, totalHeapSize=21248, min timestamp=1731348967862, max timestamp=1731348967903Num uniques 1052; 2024-11-11T18:16:07,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=202, cellSize=9250, totalHeapSize=21248, min timestamp=1731348967862, max timestamp=1731348967903Num uniques 1052; 2024-11-11T18:16:07,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48180, new segment data size=9250, suffix heap size=137376,new segment heap size=21248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=202 2024-11-11T18:16:07,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48180, new segment data size=9250, suffix heap size=137376,new segment heap size=21248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=202 2024-11-11T18:16:07,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=202, cellSize=9250, totalHeapSize=21248, min timestamp=1731348967862, max timestamp=1731348967903Num uniques 1052; 2024-11-11T18:16:07,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48180, new segment data size=9250, suffix heap size=137376,new segment heap size=21248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1052, new segment cells count=202 2024-11-11T18:16:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/40a8e996c9cd4bc1bbded829bf4dbde6 2024-11-11T18:16:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/8c005ca773a4418e8f1fd2effafe48ab is 50, key is test_row_0/C:col10/1731348964706/Put/seqid=0 2024-11-11T18:16:07,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=261, cellSize=11946, totalHeapSize=27384, min timestamp=1731348967924, max timestamp=1731348967933Num uniques 1061; 2024-11-11T18:16:07,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48586, new segment data size=11946, suffix heap size=138312,new segment heap size=27384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1061, new segment cells count=261 2024-11-11T18:16:07,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=261, cellSize=11946, totalHeapSize=27384, min timestamp=1731348967924, max timestamp=1731348967933Num uniques 1061; 2024-11-11T18:16:07,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48586, new segment data size=11946, suffix heap size=138312,new segment heap size=27384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1061, new segment cells count=261 2024-11-11T18:16:07,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=261, cellSize=11946, totalHeapSize=27384, min timestamp=1731348967924, max timestamp=1731348967933Num uniques 1061; 2024-11-11T18:16:07,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48586, new segment data size=11946, suffix heap size=138312,new segment heap size=27384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1061, new segment cells count=261 2024-11-11T18:16:07,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742411_1587 (size=9757) 2024-11-11T18:16:07,945 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/8c005ca773a4418e8f1fd2effafe48ab 2024-11-11T18:16:07,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/d5f4bce6d0064dcabc3ba740ac3b0019 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019 2024-11-11T18:16:07,954 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019, entries=100, sequenceid=195, filesize=21.9 K 2024-11-11T18:16:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/40a8e996c9cd4bc1bbded829bf4dbde6 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/40a8e996c9cd4bc1bbded829bf4dbde6 2024-11-11T18:16:07,961 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/40a8e996c9cd4bc1bbded829bf4dbde6, entries=100, sequenceid=195, filesize=9.5 K 2024-11-11T18:16:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/8c005ca773a4418e8f1fd2effafe48ab as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/8c005ca773a4418e8f1fd2effafe48ab 2024-11-11T18:16:07,967 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/8c005ca773a4418e8f1fd2effafe48ab, entries=100, sequenceid=195, filesize=9.5 K 2024-11-11T18:16:07,968 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=127.47 KB/130530 for fba45d19b0840db7bf05326b8999512f in 1289ms, sequenceid=195, compaction requested=false 2024-11-11T18:16:07,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:07,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:07,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-11T18:16:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-11T18:16:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:07,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=134.18 KB heapSize=375.42 KB 2024-11-11T18:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:07,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:07,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T18:16:07,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4430 sec 2024-11-11T18:16:07,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4490 sec 2024-11-11T18:16:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349027984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349027985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349027986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349027987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349027995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d9eeae470f824eefa58a09197d04fd91_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:08,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742412_1588 (size=14994) 2024-11-11T18:16:08,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:08,021 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d9eeae470f824eefa58a09197d04fd91_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9eeae470f824eefa58a09197d04fd91_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:08,022 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/9b85575ce6b1434f911699b641844746, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:08,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/9b85575ce6b1434f911699b641844746 is 175, key is test_row_0/A:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:08,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742413_1589 (size=39949) 2024-11-11T18:16:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349028094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349028105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349028105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349028105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349028304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349028305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349028314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349028314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349028314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=712, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/9b85575ce6b1434f911699b641844746 2024-11-11T18:16:08,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/1e22badf9ed24e27ac03de1909ec5c70 is 50, key is test_row_0/B:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:08,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742414_1590 (size=12301) 2024-11-11T18:16:08,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349028614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349028625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349028625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349028625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T18:16:08,675 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:08,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:16:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-11T18:16:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T18:16:08,677 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:16:08,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:16:08,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:16:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T18:16:08,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:08,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349028815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:08,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:08,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T18:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:08,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:08,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/1e22badf9ed24e27ac03de1909ec5c70 2024-11-11T18:16:08,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c0fa92c5550f4e45a568ee7b8ec41a9b is 50, key is test_row_0/C:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:08,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742415_1591 (size=12301) 2024-11-11T18:16:08,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T18:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:08,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:08,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T18:16:09,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T18:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:09,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:09,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349029134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349029134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349029134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349029134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:09,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=712 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c0fa92c5550f4e45a568ee7b8ec41a9b 2024-11-11T18:16:09,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/9b85575ce6b1434f911699b641844746 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746 2024-11-11T18:16:09,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746, entries=200, sequenceid=712, filesize=39.0 K 2024-11-11T18:16:09,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/1e22badf9ed24e27ac03de1909ec5c70 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1e22badf9ed24e27ac03de1909ec5c70 2024-11-11T18:16:09,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1e22badf9ed24e27ac03de1909ec5c70, entries=150, sequenceid=712, filesize=12.0 K 2024-11-11T18:16:09,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c0fa92c5550f4e45a568ee7b8ec41a9b as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c0fa92c5550f4e45a568ee7b8ec41a9b 2024-11-11T18:16:09,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c0fa92c5550f4e45a568ee7b8ec41a9b, entries=150, sequenceid=712, filesize=12.0 K 2024-11-11T18:16:09,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.30 KB/404784, currentSize=40.25 KB/41220 for fba45d19b0840db7bf05326b8999512f in 1311ms, sequenceid=712, compaction requested=true 2024-11-11T18:16:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:09,283 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:09,284 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:09,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:09,285 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:09,285 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:09,285 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,285 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1cde7e3948d54c1f9a306885bbaad45d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=91.7 K 2024-11-11T18:16:09,285 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,285 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1cde7e3948d54c1f9a306885bbaad45d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746] 2024-11-11T18:16:09,286 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cde7e3948d54c1f9a306885bbaad45d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731348963546 2024-11-11T18:16:09,286 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:09,287 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:09,287 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,287 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/8ebe0de73a7b4018a213f71f34eb0d7f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/40a8e996c9cd4bc1bbded829bf4dbde6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1e22badf9ed24e27ac03de1909ec5c70] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=33.7 K 2024-11-11T18:16:09,287 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f4bce6d0064dcabc3ba740ac3b0019, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348964701 2024-11-11T18:16:09,287 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebe0de73a7b4018a213f71f34eb0d7f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731348963546 2024-11-11T18:16:09,287 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b85575ce6b1434f911699b641844746, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1731348967960 2024-11-11T18:16:09,290 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a8e996c9cd4bc1bbded829bf4dbde6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348964701 2024-11-11T18:16:09,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,291 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:09,294 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e22badf9ed24e27ac03de1909ec5c70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1731348967964 2024-11-11T18:16:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T18:16:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a1b857716326412291df1cbfafa06677_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348967984/Put/seqid=0 2024-11-11T18:16:09,305 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:09,308 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:09,309 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/53e97100b30d484b8d88d3a81f4f9da9 is 50, key is test_row_0/B:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:09,310 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111184c771b0db1246c492e5d30f80e8abe6_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:09,312 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111184c771b0db1246c492e5d30f80e8abe6_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:09,312 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111184c771b0db1246c492e5d30f80e8abe6_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:09,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742416_1592 (size=12454) 2024-11-11T18:16:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742417_1593 (size=4469) 2024-11-11T18:16:09,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742418_1594 (size=12745) 2024-11-11T18:16:09,351 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#502 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:09,352 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/9351602a71a54302ac7b67edd592abd5 is 175, key is test_row_0/A:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:09,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742419_1595 (size=31699) 2024-11-11T18:16:09,362 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/53e97100b30d484b8d88d3a81f4f9da9 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/53e97100b30d484b8d88d3a81f4f9da9 2024-11-11T18:16:09,379 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into 53e97100b30d484b8d88d3a81f4f9da9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:09,379 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:09,379 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=13, startTime=1731348969284; duration=0sec 2024-11-11T18:16:09,379 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:09,379 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:09,379 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:09,380 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/9351602a71a54302ac7b67edd592abd5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9351602a71a54302ac7b67edd592abd5 2024-11-11T18:16:09,380 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:09,380 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/C is initiating minor compaction (all files) 2024-11-11T18:16:09,380 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/C in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:09,380 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d7d80c86a2664b34902f61da21f98b10, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/8c005ca773a4418e8f1fd2effafe48ab, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c0fa92c5550f4e45a568ee7b8ec41a9b] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=33.7 K 2024-11-11T18:16:09,380 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d80c86a2664b34902f61da21f98b10, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731348963546 2024-11-11T18:16:09,381 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c005ca773a4418e8f1fd2effafe48ab, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731348964701 2024-11-11T18:16:09,381 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0fa92c5550f4e45a568ee7b8ec41a9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1731348967964 2024-11-11T18:16:09,385 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into 9351602a71a54302ac7b67edd592abd5(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:09,385 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:09,385 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=13, startTime=1731348969283; duration=0sec 2024-11-11T18:16:09,385 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:09,385 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:09,387 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:09,388 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c1d5f71b84cb42e8b5d2b937cbd36406 is 50, key is test_row_0/C:col10/1731348967969/Put/seqid=0 2024-11-11T18:16:09,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742420_1596 (size=12745) 2024-11-11T18:16:09,408 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c1d5f71b84cb42e8b5d2b937cbd36406 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c1d5f71b84cb42e8b5d2b937cbd36406 2024-11-11T18:16:09,413 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/C of fba45d19b0840db7bf05326b8999512f into c1d5f71b84cb42e8b5d2b937cbd36406(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:09,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:09,413 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/C, priority=13, startTime=1731348969284; duration=0sec 2024-11-11T18:16:09,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:09,413 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:09,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:09,739 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a1b857716326412291df1cbfafa06677_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a1b857716326412291df1cbfafa06677_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:09,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ab51986136b848b6ac5057dd93297728, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ab51986136b848b6ac5057dd93297728 is 175, key is test_row_0/A:col10/1731348967984/Put/seqid=0 2024-11-11T18:16:09,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742421_1597 (size=31255) 2024-11-11T18:16:09,767 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=721, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ab51986136b848b6ac5057dd93297728 2024-11-11T18:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c233da246906460596f783c496b268c2 is 50, key is test_row_0/B:col10/1731348967984/Put/seqid=0 2024-11-11T18:16:09,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742422_1598 (size=12301) 2024-11-11T18:16:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T18:16:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:09,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:09,908 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348969885, max timestamp=1731348969905Num uniques 859; 2024-11-11T18:16:09,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348969885, max timestamp=1731348969905Num uniques 859; 2024-11-11T18:16:09,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348969885, max timestamp=1731348969905Num uniques 859; 2024-11-11T18:16:09,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:09,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:09,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:10,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348969948, max timestamp=1731348970012Num uniques 1018; 2024-11-11T18:16:10,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348969948, max timestamp=1731348970012Num uniques 1018; 2024-11-11T18:16:10,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:16:10,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:16:10,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731348969948, max timestamp=1731348970012Num uniques 1018; 2024-11-11T18:16:10,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T18:16:10,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731348970094, max timestamp=1731348970140Num uniques 1027; 2024-11-11T18:16:10,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T18:16:10,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731348970094, max timestamp=1731348970140Num uniques 1027; 2024-11-11T18:16:10,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T18:16:10,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731348970094, max timestamp=1731348970140Num uniques 1027; 2024-11-11T18:16:10,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T18:16:10,185 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c233da246906460596f783c496b268c2 2024-11-11T18:16:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495 is 50, key is test_row_0/C:col10/1731348967984/Put/seqid=0 2024-11-11T18:16:10,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731348970178, max timestamp=1731348970190Num uniques 1036; 2024-11-11T18:16:10,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T18:16:10,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731348970178, max timestamp=1731348970190Num uniques 1036; 2024-11-11T18:16:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349030180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T18:16:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731348970178, max timestamp=1731348970188Num uniques 1036; 2024-11-11T18:16:10,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T18:16:10,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742423_1599 (size=12301) 2024-11-11T18:16:10,209 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=721 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495 2024-11-11T18:16:10,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ab51986136b848b6ac5057dd93297728 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728 2024-11-11T18:16:10,222 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728, entries=150, sequenceid=721, filesize=30.5 K 2024-11-11T18:16:10,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c233da246906460596f783c496b268c2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c233da246906460596f783c496b268c2 2024-11-11T18:16:10,228 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c233da246906460596f783c496b268c2, entries=150, sequenceid=721, filesize=12.0 K 2024-11-11T18:16:10,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495 2024-11-11T18:16:10,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731348970217, max timestamp=1731348970226Num uniques 1045; 2024-11-11T18:16:10,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T18:16:10,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731348970217, max timestamp=1731348970226Num uniques 1045; 2024-11-11T18:16:10,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T18:16:10,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731348970217, max timestamp=1731348970226Num uniques 1045; 2024-11-11T18:16:10,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-11T18:16:10,232 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495, entries=150, sequenceid=721, filesize=12.0 K 2024-11-11T18:16:10,233 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=33.54 KB/34350 for fba45d19b0840db7bf05326b8999512f in 942ms, sequenceid=721, compaction requested=false 2024-11-11T18:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-11T18:16:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-11T18:16:10,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T18:16:10,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-11T18:16:10,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.5630 sec 2024-11-11T18:16:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:10,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=60.38 KB heapSize=157.78 KB 2024-11-11T18:16:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:10,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111db003d2b15e463280188ad3ca61c11e_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:10,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731348970277, max timestamp=1731348970284Num uniques 859; 2024-11-11T18:16:10,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T18:16:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349030281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348970277, max timestamp=1731348970284Num uniques 859; 2024-11-11T18:16:10,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:10,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348970277, max timestamp=1731348970284Num uniques 859; 2024-11-11T18:16:10,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:10,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742424_1600 (size=12454) 2024-11-11T18:16:10,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349030319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349030319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349030326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349030327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349030394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349030435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349030435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349030435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349030435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349030604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349030644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349030645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349030645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349030645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:10,712 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111db003d2b15e463280188ad3ca61c11e_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111db003d2b15e463280188ad3ca61c11e_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:10,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/94a8a12ae1f8401f9a98d4103e28c2fb, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:10,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/94a8a12ae1f8401f9a98d4103e28c2fb is 175, key is test_row_0/A:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:10,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742425_1601 (size=31255) 2024-11-11T18:16:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T18:16:10,825 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:10,825 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:16:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-11T18:16:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:10,827 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:16:10,828 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:16:10,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:16:10,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349030914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349030954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349030954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349030954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349030964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:10,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T18:16:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:10,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:10,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,119 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=817, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/94a8a12ae1f8401f9a98d4103e28c2fb 2024-11-11T18:16:11,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/214871ac0f16494ead58f44cd5662a06 is 50, key is test_row_0/B:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:11,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T18:16:11,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:11,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:11,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742426_1602 (size=12301) 2024-11-11T18:16:11,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/214871ac0f16494ead58f44cd5662a06 2024-11-11T18:16:11,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/cb2fbab6f5c542f6830eb346bd28bfaf is 50, key is test_row_0/C:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:11,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742427_1603 (size=12301) 2024-11-11T18:16:11,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T18:16:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349031424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:11,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T18:16:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349031464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349031464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349031465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349031475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:11,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=817 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/cb2fbab6f5c542f6830eb346bd28bfaf 2024-11-11T18:16:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/94a8a12ae1f8401f9a98d4103e28c2fb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb 2024-11-11T18:16:11,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb, entries=150, sequenceid=817, filesize=30.5 K 2024-11-11T18:16:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/214871ac0f16494ead58f44cd5662a06 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/214871ac0f16494ead58f44cd5662a06 2024-11-11T18:16:11,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/214871ac0f16494ead58f44cd5662a06, entries=150, sequenceid=817, filesize=12.0 K 2024-11-11T18:16:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/cb2fbab6f5c542f6830eb346bd28bfaf as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/cb2fbab6f5c542f6830eb346bd28bfaf 2024-11-11T18:16:11,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T18:16:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:11,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/cb2fbab6f5c542f6830eb346bd28bfaf, entries=150, sequenceid=817, filesize=12.0 K 2024-11-11T18:16:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~157.73 KB/161520, currentSize=136.42 KB/139690 for fba45d19b0840db7bf05326b8999512f in 1356ms, sequenceid=817, compaction requested=true 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:11,598 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:11,598 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:11,599 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:11,599 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:11,599 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,599 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9351602a71a54302ac7b67edd592abd5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=92.0 K 2024-11-11T18:16:11,599 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,599 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9351602a71a54302ac7b67edd592abd5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb] 2024-11-11T18:16:11,600 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:11,600 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9351602a71a54302ac7b67edd592abd5, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1731348967964 2024-11-11T18:16:11,600 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:11,600 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,600 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/53e97100b30d484b8d88d3a81f4f9da9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c233da246906460596f783c496b268c2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/214871ac0f16494ead58f44cd5662a06] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=36.5 K 2024-11-11T18:16:11,600 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab51986136b848b6ac5057dd93297728, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=721, earliestPutTs=1731348967983 2024-11-11T18:16:11,600 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e97100b30d484b8d88d3a81f4f9da9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1731348967964 2024-11-11T18:16:11,602 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c233da246906460596f783c496b268c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=721, earliestPutTs=1731348967983 2024-11-11T18:16:11,602 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a8a12ae1f8401f9a98d4103e28c2fb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731348970224 2024-11-11T18:16:11,602 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 214871ac0f16494ead58f44cd5662a06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731348970224 2024-11-11T18:16:11,618 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:11,619 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:11,620 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/e85309c371c9496095f6b7817e041206 is 50, key is test_row_0/B:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:11,621 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111094e13e862d44e0a8e97ab53a5d2e4e_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:11,623 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111094e13e862d44e0a8e97ab53a5d2e4e_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:11,623 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111094e13e862d44e0a8e97ab53a5d2e4e_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:11,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742428_1604 (size=12847) 2024-11-11T18:16:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742429_1605 (size=4469) 2024-11-11T18:16:11,650 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/e85309c371c9496095f6b7817e041206 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e85309c371c9496095f6b7817e041206 2024-11-11T18:16:11,652 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#511 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:11,653 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f5dc36d494544b2491639721d04eb011 is 175, key is test_row_0/A:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:11,658 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into e85309c371c9496095f6b7817e041206(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:11,658 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:11,658 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=13, startTime=1731348971598; duration=0sec 2024-11-11T18:16:11,658 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:11,658 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:11,658 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:11,661 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:11,661 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/C is initiating minor compaction (all files) 2024-11-11T18:16:11,662 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/C in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,662 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c1d5f71b84cb42e8b5d2b937cbd36406, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/cb2fbab6f5c542f6830eb346bd28bfaf] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=36.5 K 2024-11-11T18:16:11,662 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d5f71b84cb42e8b5d2b937cbd36406, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=712, earliestPutTs=1731348967964 2024-11-11T18:16:11,662 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a0dd0d0a2c0e463c89c8a8f4fd9f2495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=721, earliestPutTs=1731348967983 2024-11-11T18:16:11,662 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2fbab6f5c542f6830eb346bd28bfaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731348970224 2024-11-11T18:16:11,669 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:11,670 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a05ae8e3dd3343f689ab6e9d571b5540 is 50, key is test_row_0/C:col10/1731348970240/Put/seqid=0 2024-11-11T18:16:11,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742430_1606 (size=31801) 2024-11-11T18:16:11,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742431_1607 (size=12847) 2024-11-11T18:16:11,709 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a05ae8e3dd3343f689ab6e9d571b5540 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a05ae8e3dd3343f689ab6e9d571b5540 2024-11-11T18:16:11,716 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/C of fba45d19b0840db7bf05326b8999512f into a05ae8e3dd3343f689ab6e9d571b5540(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:11,716 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:11,716 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/C, priority=13, startTime=1731348971598; duration=0sec 2024-11-11T18:16:11,716 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:11,716 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:11,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T18:16:11,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:11,746 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-11T18:16:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:11,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111633ab1a44266482daa85d49ff549d4b8_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348970309/Put/seqid=0 2024-11-11T18:16:11,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742432_1608 (size=12454) 2024-11-11T18:16:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:12,080 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/f5dc36d494544b2491639721d04eb011 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f5dc36d494544b2491639721d04eb011 2024-11-11T18:16:12,087 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into f5dc36d494544b2491639721d04eb011(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:12,087 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:12,087 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=13, startTime=1731348971598; duration=0sec 2024-11-11T18:16:12,088 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:12,088 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:12,175 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111633ab1a44266482daa85d49ff549d4b8_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111633ab1a44266482daa85d49ff549d4b8_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:12,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ef972e218a0043e993f887e06a9143da, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ef972e218a0043e993f887e06a9143da is 175, key is test_row_0/A:col10/1731348970309/Put/seqid=0 2024-11-11T18:16:12,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742433_1609 (size=31255) 2024-11-11T18:16:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:12,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:12,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349032485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349032485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349032485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,487 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,488 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:12,488 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349032486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,491 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349032494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,497 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:12,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349032594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,608 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=856, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ef972e218a0043e993f887e06a9143da 2024-11-11T18:16:12,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c776c88fcdc148f7a2be99e75bae0ea5 is 50, key is test_row_0/B:col10/1731348970309/Put/seqid=0 2024-11-11T18:16:12,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742434_1610 (size=12301) 2024-11-11T18:16:12,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349032805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:13,033 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c776c88fcdc148f7a2be99e75bae0ea5 2024-11-11T18:16:13,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/2a8b6af71bc14def98f60078dda394d1 is 50, key is test_row_0/C:col10/1731348970309/Put/seqid=0 2024-11-11T18:16:13,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742435_1611 (size=12301) 2024-11-11T18:16:13,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:13,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349033124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:13,444 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=856 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/2a8b6af71bc14def98f60078dda394d1 2024-11-11T18:16:13,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/ef972e218a0043e993f887e06a9143da as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da 2024-11-11T18:16:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da, entries=150, sequenceid=856, filesize=30.5 K 2024-11-11T18:16:13,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c776c88fcdc148f7a2be99e75bae0ea5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c776c88fcdc148f7a2be99e75bae0ea5 2024-11-11T18:16:13,454 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c776c88fcdc148f7a2be99e75bae0ea5, entries=150, sequenceid=856, filesize=12.0 K 2024-11-11T18:16:13,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/2a8b6af71bc14def98f60078dda394d1 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/2a8b6af71bc14def98f60078dda394d1 2024-11-11T18:16:13,458 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/2a8b6af71bc14def98f60078dda394d1, entries=150, sequenceid=856, filesize=12.0 K 2024-11-11T18:16:13,459 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for fba45d19b0840db7bf05326b8999512f in 1712ms, sequenceid=856, compaction requested=false 2024-11-11T18:16:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-11T18:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-11T18:16:13,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-11T18:16:13,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-11T18:16:13,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.6370 sec 2024-11-11T18:16:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:13,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T18:16:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:13,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:13,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118d410df912344e438462e8f12af37be2_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:13,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742436_1612 (size=14994) 2024-11-11T18:16:13,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:13,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118d410df912344e438462e8f12af37be2_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118d410df912344e438462e8f12af37be2_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:13,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b9ee14b790be482092ed571a596e1ac4, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:13,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b9ee14b790be482092ed571a596e1ac4 is 175, key is test_row_0/A:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:13,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742437_1613 (size=39949) 2024-11-11T18:16:13,664 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=868, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b9ee14b790be482092ed571a596e1ac4 2024-11-11T18:16:13,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/e79917bf45014c198bfcd3daf9ee7058 is 50, key is test_row_0/B:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:13,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742438_1614 (size=12301) 2024-11-11T18:16:13,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=868 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/e79917bf45014c198bfcd3daf9ee7058 2024-11-11T18:16:13,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/b64fb89d5fa0484990fdc63eddffe71e is 50, key is test_row_0/C:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:13,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742439_1615 (size=12301) 2024-11-11T18:16:13,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348973695, max timestamp=1731348973717Num uniques 859; 2024-11-11T18:16:13,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:13,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348973695, max timestamp=1731348973717Num uniques 859; 2024-11-11T18:16:13,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348973695, max timestamp=1731348973717Num uniques 859; 2024-11-11T18:16:13,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:13,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:13,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349033799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:13,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349033914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=868 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/b64fb89d5fa0484990fdc63eddffe71e 2024-11-11T18:16:14,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/b9ee14b790be482092ed571a596e1ac4 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4 2024-11-11T18:16:14,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4, entries=200, sequenceid=868, filesize=39.0 K 2024-11-11T18:16:14,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/e79917bf45014c198bfcd3daf9ee7058 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e79917bf45014c198bfcd3daf9ee7058 2024-11-11T18:16:14,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e79917bf45014c198bfcd3daf9ee7058, entries=150, sequenceid=868, filesize=12.0 K 2024-11-11T18:16:14,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/b64fb89d5fa0484990fdc63eddffe71e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b64fb89d5fa0484990fdc63eddffe71e 2024-11-11T18:16:14,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b64fb89d5fa0484990fdc63eddffe71e, entries=150, sequenceid=868, filesize=12.0 K 2024-11-11T18:16:14,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for fba45d19b0840db7bf05326b8999512f in 460ms, sequenceid=868, compaction requested=true 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:14,097 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:14,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:14,097 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:14,098 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:14,098 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:14,098 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:14,098 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:14,098 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:14,098 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:14,098 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e85309c371c9496095f6b7817e041206, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c776c88fcdc148f7a2be99e75bae0ea5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e79917bf45014c198bfcd3daf9ee7058] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=36.6 K 2024-11-11T18:16:14,098 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f5dc36d494544b2491639721d04eb011, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=100.6 K 2024-11-11T18:16:14,098 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:14,098 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f5dc36d494544b2491639721d04eb011, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4] 2024-11-11T18:16:14,099 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e85309c371c9496095f6b7817e041206, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731348970224 2024-11-11T18:16:14,099 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5dc36d494544b2491639721d04eb011, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731348970224 2024-11-11T18:16:14,099 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef972e218a0043e993f887e06a9143da, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731348970309 2024-11-11T18:16:14,099 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c776c88fcdc148f7a2be99e75bae0ea5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731348970309 2024-11-11T18:16:14,099 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting e79917bf45014c198bfcd3daf9ee7058, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=868, earliestPutTs=1731348972452 2024-11-11T18:16:14,099 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ee14b790be482092ed571a596e1ac4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=868, earliestPutTs=1731348972452 2024-11-11T18:16:14,104 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:14,105 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:14,105 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/5d493bb308734100aba9b8fb61b49080 is 50, key is test_row_0/B:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:14,106 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111d292a1ff5ce7465b97484ace14232f3c_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:14,107 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111d292a1ff5ce7465b97484ace14232f3c_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:14,107 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d292a1ff5ce7465b97484ace14232f3c_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:14,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742440_1616 (size=12949) 2024-11-11T18:16:14,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742441_1617 (size=4469) 2024-11-11T18:16:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:14,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T18:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:14,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T18:16:14,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111140f5d02ad8a54a3088f51741a1e02cfd_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348973786/Put/seqid=0 2024-11-11T18:16:14,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742442_1618 (size=14994) 2024-11-11T18:16:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349034173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349034284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35344 deadline: 1731349034495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349034495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_0' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35382 deadline: 1731349034504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,507 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_1' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,514 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/5d493bb308734100aba9b8fb61b49080 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/5d493bb308734100aba9b8fb61b49080 2024-11-11T18:16:14,515 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#520 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:14,516 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/780dbac8b0a04d6ca6b0956e9a1e417f is 175, key is test_row_0/A:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:14,518 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into 5d493bb308734100aba9b8fb61b49080(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:14,518 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:14,518 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=13, startTime=1731348974097; duration=0sec 2024-11-11T18:16:14,518 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:14,518 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:14,518 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T18:16:14,519 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T18:16:14,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,519 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/C is initiating minor compaction (all files) 2024-11-11T18:16:14,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742443_1619 (size=31903) 2024-11-11T18:16:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35366 deadline: 1731349034514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,520 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/C in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:14,520 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a05ae8e3dd3343f689ab6e9d571b5540, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/2a8b6af71bc14def98f60078dda394d1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b64fb89d5fa0484990fdc63eddffe71e] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=36.6 K 2024-11-11T18:16:14,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35336 deadline: 1731349034514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,520 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting a05ae8e3dd3343f689ab6e9d571b5540, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=817, earliestPutTs=1731348970224 2024-11-11T18:16:14,520 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8b6af71bc14def98f60078dda394d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=856, earliestPutTs=1731348970309 2024-11-11T18:16:14,520 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:14,521 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting b64fb89d5fa0484990fdc63eddffe71e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=868, earliestPutTs=1731348972452 2024-11-11T18:16:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,521 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 90f34c5bee6a,42957,1731348800579 for 'test_row_2' in fba45d19b0840db7bf05326b8999512f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,524 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/780dbac8b0a04d6ca6b0956e9a1e417f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/780dbac8b0a04d6ca6b0956e9a1e417f 2024-11-11T18:16:14,528 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into 780dbac8b0a04d6ca6b0956e9a1e417f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:14,528 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:14,528 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=13, startTime=1731348974097; duration=0sec 2024-11-11T18:16:14,528 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:14,528 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:14,528 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#C#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:14,529 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/4f3a6129292b41879b725eb0c4c5e4fb is 50, key is test_row_0/C:col10/1731348973636/Put/seqid=0 2024-11-11T18:16:14,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742444_1620 (size=12949) 2024-11-11T18:16:14,548 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:14,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111140f5d02ad8a54a3088f51741a1e02cfd_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111140f5d02ad8a54a3088f51741a1e02cfd_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:14,552 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/89aed461958041c19d43c954ab496587, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:14,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/89aed461958041c19d43c954ab496587 is 175, key is test_row_0/A:col10/1731348973786/Put/seqid=0 2024-11-11T18:16:14,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742445_1621 (size=39949) 2024-11-11T18:16:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349034804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:14,937 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/4f3a6129292b41879b725eb0c4c5e4fb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/4f3a6129292b41879b725eb0c4c5e4fb 2024-11-11T18:16:14,940 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fba45d19b0840db7bf05326b8999512f/C of fba45d19b0840db7bf05326b8999512f into 4f3a6129292b41879b725eb0c4c5e4fb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:14,940 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:14,940 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/C, priority=13, startTime=1731348974097; duration=0sec 2024-11-11T18:16:14,941 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:14,941 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:14,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=905, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/89aed461958041c19d43c954ab496587 2024-11-11T18:16:14,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c62f78aabc5b4ef8902c6e1eab46b9cb is 50, key is test_row_0/B:col10/1731348973786/Put/seqid=0 2024-11-11T18:16:14,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742446_1622 (size=12301) 2024-11-11T18:16:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T18:16:14,994 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:14,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T18:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-11T18:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:14,996 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T18:16:14,997 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T18:16:14,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T18:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:15,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T18:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T18:16:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349035314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:15,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=905 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c62f78aabc5b4ef8902c6e1eab46b9cb 2024-11-11T18:16:15,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/3cfe9f22eed043b08761107db85465f8 is 50, key is test_row_0/C:col10/1731348973786/Put/seqid=0 2024-11-11T18:16:15,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742447_1623 (size=12301) 2024-11-11T18:16:15,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T18:16:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T18:16:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:15,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:15,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T18:16:15,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:15,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T18:16:15,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=905 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/3cfe9f22eed043b08761107db85465f8 2024-11-11T18:16:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/89aed461958041c19d43c954ab496587 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587 2024-11-11T18:16:15,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587, entries=200, sequenceid=905, filesize=39.0 K 2024-11-11T18:16:15,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/c62f78aabc5b4ef8902c6e1eab46b9cb as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c62f78aabc5b4ef8902c6e1eab46b9cb 2024-11-11T18:16:15,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c62f78aabc5b4ef8902c6e1eab46b9cb, entries=150, sequenceid=905, filesize=12.0 K 2024-11-11T18:16:15,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/3cfe9f22eed043b08761107db85465f8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3cfe9f22eed043b08761107db85465f8 2024-11-11T18:16:15,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3cfe9f22eed043b08761107db85465f8, entries=150, sequenceid=905, filesize=12.0 K 2024-11-11T18:16:15,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fba45d19b0840db7bf05326b8999512f in 1665ms, sequenceid=905, compaction requested=false 2024-11-11T18:16:15,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:15,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42957 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T18:16:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:15,915 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T18:16:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:15,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:15,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b778b4f8a3a940b699a16a1e74f848e3_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348974145/Put/seqid=0 2024-11-11T18:16:15,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742448_1624 (size=12454) 2024-11-11T18:16:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:16,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:16,331 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b778b4f8a3a940b699a16a1e74f848e3_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b778b4f8a3a940b699a16a1e74f848e3_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:16,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/51b87e60e4eb49fba8c55aa55a914981, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:16,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/51b87e60e4eb49fba8c55aa55a914981 is 175, key is test_row_0/A:col10/1731348974145/Put/seqid=0 2024-11-11T18:16:16,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. as already flushing 2024-11-11T18:16:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:16,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742449_1625 (size=31255) 2024-11-11T18:16:16,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348976404, max timestamp=1731348976422Num uniques 859; 2024-11-11T18:16:16,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:16,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348976404, max timestamp=1731348976422Num uniques 859; 2024-11-11T18:16:16,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:16,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731348976404, max timestamp=1731348976422Num uniques 859; 2024-11-11T18:16:16,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/90f34c5bee6a:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T18:16:16,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349036514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349036624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:16,737 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=918, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/51b87e60e4eb49fba8c55aa55a914981 2024-11-11T18:16:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/0b65112dfff14b6987fdbea2118d8651 is 50, key is test_row_0/B:col10/1731348974145/Put/seqid=0 2024-11-11T18:16:16,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742450_1626 (size=12301) 2024-11-11T18:16:16,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:16,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349036835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:16,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:17,050 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-11T18:16:17,050 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:17,050 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,050 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:17,051 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-11T18:16:17,051 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:17,051 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,051 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:17,052 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-11T18:16:17,052 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:17,052 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,052 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:17,055 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-11T18:16:17,055 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:17,055 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,055 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-11T18:16:17,055 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,055 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:17,055 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:17,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:17,055 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T18:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35396 deadline: 1731349037144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 , the old value is region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T18:16:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., hostname=90f34c5bee6a,42957,1731348800579, seqNum=5 because the exception is null or not the one we care about 2024-11-11T18:16:17,148 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=918 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/0b65112dfff14b6987fdbea2118d8651 2024-11-11T18:16:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/1b88a5e65c1041dfbba5afb405a0ef40 is 50, key is test_row_0/C:col10/1731348974145/Put/seqid=0 2024-11-11T18:16:17,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742451_1627 (size=12301) 2024-11-11T18:16:17,556 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=918 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/1b88a5e65c1041dfbba5afb405a0ef40 2024-11-11T18:16:17,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/51b87e60e4eb49fba8c55aa55a914981 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981 2024-11-11T18:16:17,562 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981, entries=150, sequenceid=918, filesize=30.5 K 2024-11-11T18:16:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/0b65112dfff14b6987fdbea2118d8651 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/0b65112dfff14b6987fdbea2118d8651 2024-11-11T18:16:17,565 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/0b65112dfff14b6987fdbea2118d8651, entries=150, sequenceid=918, filesize=12.0 K 2024-11-11T18:16:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/1b88a5e65c1041dfbba5afb405a0ef40 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/1b88a5e65c1041dfbba5afb405a0ef40 2024-11-11T18:16:17,568 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/1b88a5e65c1041dfbba5afb405a0ef40, entries=150, sequenceid=918, filesize=12.0 K 2024-11-11T18:16:17,569 INFO [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fba45d19b0840db7bf05326b8999512f in 1654ms, sequenceid=918, compaction requested=true 2024-11-11T18:16:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/90f34c5bee6a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-11T18:16:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-11T18:16:17,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-11T18:16:17,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5730 sec 2024-11-11T18:16:17,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.5760 sec 2024-11-11T18:16:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42957 {}] regionserver.HRegion(8855): Flush requested on fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:17,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T18:16:17,656 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-11T18:16:17,656 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:17,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:17,656 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:16:17,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:17,657 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:17,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:16:17,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:17,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T18:16:17,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:17,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111150365135374e46dabce6dd841ec8462e_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348977655/Put/seqid=0 2024-11-11T18:16:17,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742452_1628 (size=12454) 2024-11-11T18:16:18,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:18,068 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111150365135374e46dabce6dd841ec8462e_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111150365135374e46dabce6dd841ec8462e_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:18,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/4ca1d4fd61c3410d835d713b3bf64e48, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:18,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/4ca1d4fd61c3410d835d713b3bf64e48 is 175, key is test_row_0/A:col10/1731348977655/Put/seqid=0 2024-11-11T18:16:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742453_1629 (size=31255) 2024-11-11T18:16:18,473 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=956, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/4ca1d4fd61c3410d835d713b3bf64e48 2024-11-11T18:16:18,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/fd573d15a61d4349b8da3996523936a0 is 50, key is test_row_0/B:col10/1731348977655/Put/seqid=0 2024-11-11T18:16:18,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742454_1630 (size=12301) 2024-11-11T18:16:18,536 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-11T18:16:18,536 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:18,536 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,536 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:18,545 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-11T18:16:18,545 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:18,545 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,545 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:18,555 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-11T18:16:18,555 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:18,555 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,555 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:18,565 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-11T18:16:18,565 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T18:16:18,565 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,565 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:18,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:18,810 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T18:16:18,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=956 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/fd573d15a61d4349b8da3996523936a0 2024-11-11T18:16:18,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a453b3c9ef154796a2b69d8f7943b0ea is 50, key is test_row_0/C:col10/1731348977655/Put/seqid=0 2024-11-11T18:16:18,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742455_1631 (size=12301) 2024-11-11T18:16:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T18:16:19,155 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 240 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 171 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 176 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2987 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2958 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2640 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2991 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2958 2024-11-11T18:16:19,155 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T18:16:19,155 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:16:19,155 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:16:19,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:19,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:19,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:19,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T18:16:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:19,158 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348979158"}]},"ts":"1731348979158"} 2024-11-11T18:16:19,159 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T18:16:19,159 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T18:16:19,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T18:16:19,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, UNASSIGN}] 2024-11-11T18:16:19,162 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, UNASSIGN 2024-11-11T18:16:19,162 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=CLOSING, regionLocation=90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:19,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, UNASSIGN because future has completed 2024-11-11T18:16:19,163 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T18:16:19,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579}] 2024-11-11T18:16:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:19,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=956 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a453b3c9ef154796a2b69d8f7943b0ea 2024-11-11T18:16:19,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/4ca1d4fd61c3410d835d713b3bf64e48 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48 2024-11-11T18:16:19,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48, entries=150, sequenceid=956, filesize=30.5 K 2024-11-11T18:16:19,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/fd573d15a61d4349b8da3996523936a0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/fd573d15a61d4349b8da3996523936a0 2024-11-11T18:16:19,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/fd573d15a61d4349b8da3996523936a0, entries=150, sequenceid=956, filesize=12.0 K 2024-11-11T18:16:19,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/a453b3c9ef154796a2b69d8f7943b0ea as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a453b3c9ef154796a2b69d8f7943b0ea 2024-11-11T18:16:19,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a453b3c9ef154796a2b69d8f7943b0ea, entries=150, sequenceid=956, filesize=12.0 K 2024-11-11T18:16:19,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=26.84 KB/27480 for fba45d19b0840db7bf05326b8999512f in 1647ms, sequenceid=956, compaction requested=true 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:19,303 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fba45d19b0840db7bf05326b8999512f:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T18:16:19,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:19,303 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/B is initiating minor compaction (all files) 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1541): fba45d19b0840db7bf05326b8999512f/A is initiating minor compaction (all files) 2024-11-11T18:16:19,304 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/B in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:19,304 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fba45d19b0840db7bf05326b8999512f/A in TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:19,304 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/5d493bb308734100aba9b8fb61b49080, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c62f78aabc5b4ef8902c6e1eab46b9cb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/0b65112dfff14b6987fdbea2118d8651, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/fd573d15a61d4349b8da3996523936a0] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=48.7 K 2024-11-11T18:16:19,304 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/780dbac8b0a04d6ca6b0956e9a1e417f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48] into tmpdir=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp, totalSize=131.2 K 2024-11-11T18:16:19,304 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. files: [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/780dbac8b0a04d6ca6b0956e9a1e417f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48] 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d493bb308734100aba9b8fb61b49080, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=868, earliestPutTs=1731348972452 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 780dbac8b0a04d6ca6b0956e9a1e417f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=868, earliestPutTs=1731348972452 2024-11-11T18:16:19,304 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting c62f78aabc5b4ef8902c6e1eab46b9cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=905, earliestPutTs=1731348973786 2024-11-11T18:16:19,305 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89aed461958041c19d43c954ab496587, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=905, earliestPutTs=1731348973786 2024-11-11T18:16:19,305 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b65112dfff14b6987fdbea2118d8651, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=918, earliestPutTs=1731348974145 2024-11-11T18:16:19,305 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] compactions.Compactor(225): Compacting fd573d15a61d4349b8da3996523936a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=956, earliestPutTs=1731348976487 2024-11-11T18:16:19,305 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51b87e60e4eb49fba8c55aa55a914981, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=918, earliestPutTs=1731348974145 2024-11-11T18:16:19,305 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ca1d4fd61c3410d835d713b3bf64e48, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=956, earliestPutTs=1731348976487 2024-11-11T18:16:19,310 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:19,311 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:19,311 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111183b13e26e6d746ce923ae1c5eeae5286_fba45d19b0840db7bf05326b8999512f store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:19,311 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/ed75142259684ad48d33c5a4d84e1ad2 is 50, key is test_row_0/B:col10/1731348977655/Put/seqid=0 2024-11-11T18:16:19,314 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111183b13e26e6d746ce923ae1c5eeae5286_fba45d19b0840db7bf05326b8999512f, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:19,314 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111183b13e26e6d746ce923ae1c5eeae5286_fba45d19b0840db7bf05326b8999512f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:19,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742456_1632 (size=13085) 2024-11-11T18:16:19,315 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:19,315 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T18:16:19,315 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing fba45d19b0840db7bf05326b8999512f, disabling compactions & flushes 2024-11-11T18:16:19,315 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:19,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742457_1633 (size=4469) 2024-11-11T18:16:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:19,717 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fba45d19b0840db7bf05326b8999512f#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T18:16:19,718 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1ba7e04b2ef748bdadbda935bc61ac54 is 175, key is test_row_0/A:col10/1731348977655/Put/seqid=0 2024-11-11T18:16:19,718 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/ed75142259684ad48d33c5a4d84e1ad2 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ed75142259684ad48d33c5a4d84e1ad2 2024-11-11T18:16:19,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742458_1634 (size=32039) 2024-11-11T18:16:19,721 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fba45d19b0840db7bf05326b8999512f/B of fba45d19b0840db7bf05326b8999512f into ed75142259684ad48d33c5a4d84e1ad2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:19,721 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:19,722 INFO [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/B, priority=12, startTime=1731348979303; duration=0sec 2024-11-11T18:16:19,722 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T18:16:19,722 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:B 2024-11-11T18:16:19,722 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. because compaction request was cancelled 2024-11-11T18:16:19,722 DEBUG [RS:0;90f34c5bee6a:42957-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:C 2024-11-11T18:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:20,125 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/1ba7e04b2ef748bdadbda935bc61ac54 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1ba7e04b2ef748bdadbda935bc61ac54 2024-11-11T18:16:20,128 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fba45d19b0840db7bf05326b8999512f/A of fba45d19b0840db7bf05326b8999512f into 1ba7e04b2ef748bdadbda935bc61ac54(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T18:16:20,128 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fba45d19b0840db7bf05326b8999512f: 2024-11-11T18:16:20,128 INFO [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., storeName=fba45d19b0840db7bf05326b8999512f/A, priority=12, startTime=1731348979303; duration=0sec 2024-11-11T18:16:20,128 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:20,128 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T18:16:20,128 DEBUG [RS:0;90f34c5bee6a:42957-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fba45d19b0840db7bf05326b8999512f:A 2024-11-11T18:16:20,128 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:20,128 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. after waiting 0 ms 2024-11-11T18:16:20,128 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:20,128 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing fba45d19b0840db7bf05326b8999512f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T18:16:20,129 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=A 2024-11-11T18:16:20,129 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:20,129 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=B 2024-11-11T18:16:20,129 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:20,129 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fba45d19b0840db7bf05326b8999512f, store=C 2024-11-11T18:16:20,129 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T18:16:20,133 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111916d1aa2f281420bb585af3e4bbf4adc_fba45d19b0840db7bf05326b8999512f is 50, key is test_row_0/A:col10/1731348978535/Put/seqid=0 2024-11-11T18:16:20,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742459_1635 (size=12454) 2024-11-11T18:16:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:20,536 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T18:16:20,539 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111916d1aa2f281420bb585af3e4bbf4adc_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111916d1aa2f281420bb585af3e4bbf4adc_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:20,540 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/a41b06d011c5424fb532d1ae2341f23e, store: [table=TestAcidGuarantees family=A region=fba45d19b0840db7bf05326b8999512f] 2024-11-11T18:16:20,540 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/a41b06d011c5424fb532d1ae2341f23e is 175, key is test_row_0/A:col10/1731348978535/Put/seqid=0 2024-11-11T18:16:20,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742460_1636 (size=31255) 2024-11-11T18:16:20,944 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=965, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/a41b06d011c5424fb532d1ae2341f23e 2024-11-11T18:16:20,949 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/604f1fc673cf4ea4a5e00f90ac069297 is 50, key is test_row_0/B:col10/1731348978535/Put/seqid=0 2024-11-11T18:16:20,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742461_1637 (size=12301) 2024-11-11T18:16:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:21,353 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=965 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/604f1fc673cf4ea4a5e00f90ac069297 2024-11-11T18:16:21,358 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c7736b65d6fa42dabe63db42bad005f8 is 50, key is test_row_0/C:col10/1731348978535/Put/seqid=0 2024-11-11T18:16:21,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742462_1638 (size=12301) 2024-11-11T18:16:21,761 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=965 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c7736b65d6fa42dabe63db42bad005f8 2024-11-11T18:16:21,764 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/A/a41b06d011c5424fb532d1ae2341f23e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/a41b06d011c5424fb532d1ae2341f23e 2024-11-11T18:16:21,766 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/a41b06d011c5424fb532d1ae2341f23e, entries=150, sequenceid=965, filesize=30.5 K 2024-11-11T18:16:21,767 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/B/604f1fc673cf4ea4a5e00f90ac069297 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/604f1fc673cf4ea4a5e00f90ac069297 2024-11-11T18:16:21,769 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/604f1fc673cf4ea4a5e00f90ac069297, entries=150, sequenceid=965, filesize=12.0 K 2024-11-11T18:16:21,769 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/.tmp/C/c7736b65d6fa42dabe63db42bad005f8 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c7736b65d6fa42dabe63db42bad005f8 2024-11-11T18:16:21,772 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c7736b65d6fa42dabe63db42bad005f8, entries=150, sequenceid=965, filesize=12.0 K 2024-11-11T18:16:21,772 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fba45d19b0840db7bf05326b8999512f in 1644ms, sequenceid=965, compaction requested=true 2024-11-11T18:16:21,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/5f3ddc9edd694e8485e9e859b75c7962, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/88dd2152ce9f498c9f70f6fa75b5d131, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1cde7e3948d54c1f9a306885bbaad45d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9351602a71a54302ac7b67edd592abd5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f5dc36d494544b2491639721d04eb011, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/780dbac8b0a04d6ca6b0956e9a1e417f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48] to archive 2024-11-11T18:16:21,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:16:21,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/3242613ffb264876af519a95f314a654 2024-11-11T18:16:21,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1d3c4f70b68d4e40adf3503fb88fbc60 2024-11-11T18:16:21,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d340d3cdd8ce43a18010bfd9b1897bc4 2024-11-11T18:16:21,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/5f3ddc9edd694e8485e9e859b75c7962 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/5f3ddc9edd694e8485e9e859b75c7962 2024-11-11T18:16:21,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f567f9e290de4b91bbb3cc2bc1c75a41 2024-11-11T18:16:21,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/021435e6f3c04aeb841c932bde4ed204 2024-11-11T18:16:21,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/88dd2152ce9f498c9f70f6fa75b5d131 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/88dd2152ce9f498c9f70f6fa75b5d131 2024-11-11T18:16:21,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b3580e1748da4db2884d0c10dacf6ec5 2024-11-11T18:16:21,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f70d7ebb3c2245de8764031b89a262c6 2024-11-11T18:16:21,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/41980c1652a048319f1e7da2acd0156c 2024-11-11T18:16:21,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1cde7e3948d54c1f9a306885bbaad45d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1cde7e3948d54c1f9a306885bbaad45d 2024-11-11T18:16:21,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/d5f4bce6d0064dcabc3ba740ac3b0019 2024-11-11T18:16:21,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9b85575ce6b1434f911699b641844746 2024-11-11T18:16:21,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9351602a71a54302ac7b67edd592abd5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/9351602a71a54302ac7b67edd592abd5 2024-11-11T18:16:21,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ab51986136b848b6ac5057dd93297728 2024-11-11T18:16:21,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f5dc36d494544b2491639721d04eb011 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/f5dc36d494544b2491639721d04eb011 2024-11-11T18:16:21,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/94a8a12ae1f8401f9a98d4103e28c2fb 2024-11-11T18:16:21,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/ef972e218a0043e993f887e06a9143da 2024-11-11T18:16:21,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/b9ee14b790be482092ed571a596e1ac4 2024-11-11T18:16:21,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/780dbac8b0a04d6ca6b0956e9a1e417f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/780dbac8b0a04d6ca6b0956e9a1e417f 2024-11-11T18:16:21,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/89aed461958041c19d43c954ab496587 2024-11-11T18:16:21,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/51b87e60e4eb49fba8c55aa55a914981 2024-11-11T18:16:21,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/4ca1d4fd61c3410d835d713b3bf64e48 2024-11-11T18:16:21,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/dd3e68f91ca8458f94250e213d702629, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/7f8be4f197584ce8bade7dfa0dd1a44a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/18b0b9a8d2bc4e20ac54e7bfef07caf0, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/a53bea6f2f04425c99f4b3c1f1b708f5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ca33364ad6e04631b5ae20857003f0b3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1cd2590d3c3443eabaf2c99589565f85, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/24962998409e480086387c8e77e5f628, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3e99180330f0416db68663d3a138f23a, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/908a80781d4b4acebf873abaab6cd527, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/8ebe0de73a7b4018a213f71f34eb0d7f, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3276bc6ba42a432daa07f43901c5560d, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/40a8e996c9cd4bc1bbded829bf4dbde6, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/53e97100b30d484b8d88d3a81f4f9da9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1e22badf9ed24e27ac03de1909ec5c70, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c233da246906460596f783c496b268c2, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e85309c371c9496095f6b7817e041206, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/214871ac0f16494ead58f44cd5662a06, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c776c88fcdc148f7a2be99e75bae0ea5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/5d493bb308734100aba9b8fb61b49080, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e79917bf45014c198bfcd3daf9ee7058, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c62f78aabc5b4ef8902c6e1eab46b9cb, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/0b65112dfff14b6987fdbea2118d8651, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/fd573d15a61d4349b8da3996523936a0] to archive 2024-11-11T18:16:21,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:16:21,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/dd3e68f91ca8458f94250e213d702629 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/dd3e68f91ca8458f94250e213d702629 2024-11-11T18:16:21,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/7f8be4f197584ce8bade7dfa0dd1a44a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/7f8be4f197584ce8bade7dfa0dd1a44a 2024-11-11T18:16:21,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/18b0b9a8d2bc4e20ac54e7bfef07caf0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/18b0b9a8d2bc4e20ac54e7bfef07caf0 2024-11-11T18:16:21,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/a53bea6f2f04425c99f4b3c1f1b708f5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/a53bea6f2f04425c99f4b3c1f1b708f5 2024-11-11T18:16:21,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ca33364ad6e04631b5ae20857003f0b3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ca33364ad6e04631b5ae20857003f0b3 2024-11-11T18:16:21,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1cd2590d3c3443eabaf2c99589565f85 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1cd2590d3c3443eabaf2c99589565f85 2024-11-11T18:16:21,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/24962998409e480086387c8e77e5f628 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/24962998409e480086387c8e77e5f628 2024-11-11T18:16:21,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3e99180330f0416db68663d3a138f23a to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3e99180330f0416db68663d3a138f23a 2024-11-11T18:16:21,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/908a80781d4b4acebf873abaab6cd527 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/908a80781d4b4acebf873abaab6cd527 2024-11-11T18:16:21,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/8ebe0de73a7b4018a213f71f34eb0d7f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/8ebe0de73a7b4018a213f71f34eb0d7f 2024-11-11T18:16:21,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3276bc6ba42a432daa07f43901c5560d to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/3276bc6ba42a432daa07f43901c5560d 2024-11-11T18:16:21,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/40a8e996c9cd4bc1bbded829bf4dbde6 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/40a8e996c9cd4bc1bbded829bf4dbde6 2024-11-11T18:16:21,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/53e97100b30d484b8d88d3a81f4f9da9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/53e97100b30d484b8d88d3a81f4f9da9 2024-11-11T18:16:21,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1e22badf9ed24e27ac03de1909ec5c70 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/1e22badf9ed24e27ac03de1909ec5c70 2024-11-11T18:16:21,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c233da246906460596f783c496b268c2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c233da246906460596f783c496b268c2 2024-11-11T18:16:21,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e85309c371c9496095f6b7817e041206 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e85309c371c9496095f6b7817e041206 2024-11-11T18:16:21,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/214871ac0f16494ead58f44cd5662a06 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/214871ac0f16494ead58f44cd5662a06 2024-11-11T18:16:21,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c776c88fcdc148f7a2be99e75bae0ea5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c776c88fcdc148f7a2be99e75bae0ea5 2024-11-11T18:16:21,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/5d493bb308734100aba9b8fb61b49080 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/5d493bb308734100aba9b8fb61b49080 2024-11-11T18:16:21,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e79917bf45014c198bfcd3daf9ee7058 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/e79917bf45014c198bfcd3daf9ee7058 2024-11-11T18:16:21,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c62f78aabc5b4ef8902c6e1eab46b9cb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/c62f78aabc5b4ef8902c6e1eab46b9cb 2024-11-11T18:16:21,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/0b65112dfff14b6987fdbea2118d8651 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/0b65112dfff14b6987fdbea2118d8651 2024-11-11T18:16:21,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/fd573d15a61d4349b8da3996523936a0 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/fd573d15a61d4349b8da3996523936a0 2024-11-11T18:16:21,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/f4ee6c81abf6444ca4ff5041608ecfa5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/9c8e19cb3d82463e8d144f76b30d7b52, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5160016e83fd475d88f39f5a6c0519f9, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/77e1a593161940d29a4e2eab98de7da3, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3030cf68b2684a97acb20057d9e3ddb5, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d8d8d8bda5e747fca573d736a0342cf1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a3eebaaaa6b54f59abd32c455bf6c0de, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b93c8a2e3f1f4cbdb50544aac3ca4233, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5cf354d114dd4fe18c9cbcfe37eb75e4, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d7d80c86a2664b34902f61da21f98b10, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5975f2b6cb4b4623a5677730ddafbe1b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/8c005ca773a4418e8f1fd2effafe48ab, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c1d5f71b84cb42e8b5d2b937cbd36406, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c0fa92c5550f4e45a568ee7b8ec41a9b, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a05ae8e3dd3343f689ab6e9d571b5540, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/cb2fbab6f5c542f6830eb346bd28bfaf, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/2a8b6af71bc14def98f60078dda394d1, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b64fb89d5fa0484990fdc63eddffe71e] to archive 2024-11-11T18:16:21,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T18:16:21,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/f4ee6c81abf6444ca4ff5041608ecfa5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/f4ee6c81abf6444ca4ff5041608ecfa5 2024-11-11T18:16:21,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/9c8e19cb3d82463e8d144f76b30d7b52 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/9c8e19cb3d82463e8d144f76b30d7b52 2024-11-11T18:16:21,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5160016e83fd475d88f39f5a6c0519f9 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5160016e83fd475d88f39f5a6c0519f9 2024-11-11T18:16:21,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/77e1a593161940d29a4e2eab98de7da3 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/77e1a593161940d29a4e2eab98de7da3 2024-11-11T18:16:21,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3030cf68b2684a97acb20057d9e3ddb5 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3030cf68b2684a97acb20057d9e3ddb5 2024-11-11T18:16:21,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d8d8d8bda5e747fca573d736a0342cf1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d8d8d8bda5e747fca573d736a0342cf1 2024-11-11T18:16:21,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a3eebaaaa6b54f59abd32c455bf6c0de to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a3eebaaaa6b54f59abd32c455bf6c0de 2024-11-11T18:16:21,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b93c8a2e3f1f4cbdb50544aac3ca4233 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b93c8a2e3f1f4cbdb50544aac3ca4233 2024-11-11T18:16:21,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5cf354d114dd4fe18c9cbcfe37eb75e4 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5cf354d114dd4fe18c9cbcfe37eb75e4 2024-11-11T18:16:21,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d7d80c86a2664b34902f61da21f98b10 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/d7d80c86a2664b34902f61da21f98b10 2024-11-11T18:16:21,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5975f2b6cb4b4623a5677730ddafbe1b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/5975f2b6cb4b4623a5677730ddafbe1b 2024-11-11T18:16:21,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/8c005ca773a4418e8f1fd2effafe48ab to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/8c005ca773a4418e8f1fd2effafe48ab 2024-11-11T18:16:21,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c1d5f71b84cb42e8b5d2b937cbd36406 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c1d5f71b84cb42e8b5d2b937cbd36406 2024-11-11T18:16:21,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c0fa92c5550f4e45a568ee7b8ec41a9b to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c0fa92c5550f4e45a568ee7b8ec41a9b 2024-11-11T18:16:21,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a0dd0d0a2c0e463c89c8a8f4fd9f2495 2024-11-11T18:16:21,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a05ae8e3dd3343f689ab6e9d571b5540 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a05ae8e3dd3343f689ab6e9d571b5540 2024-11-11T18:16:21,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/cb2fbab6f5c542f6830eb346bd28bfaf to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/cb2fbab6f5c542f6830eb346bd28bfaf 2024-11-11T18:16:21,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/2a8b6af71bc14def98f60078dda394d1 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/2a8b6af71bc14def98f60078dda394d1 2024-11-11T18:16:21,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b64fb89d5fa0484990fdc63eddffe71e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/b64fb89d5fa0484990fdc63eddffe71e 2024-11-11T18:16:21,827 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/recovered.edits/968.seqid, newMaxSeqId=968, maxSeqId=4 2024-11-11T18:16:21,827 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. 2024-11-11T18:16:21,828 DEBUG [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for fba45d19b0840db7bf05326b8999512f: Waiting for close lock at 1731348979315Running coprocessor pre-close hooks at 1731348979315Disabling compacts and flushes for region at 1731348979315Disabling writes for close at 1731348980128 (+813 ms)Obtaining lock to block concurrent updates at 1731348980128Preparing flush snapshotting stores in fba45d19b0840db7bf05326b8999512f at 1731348980128Finished memstore snapshotting TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731348980129 (+1 ms)Flushing stores of TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f. at 1731348980129Flushing fba45d19b0840db7bf05326b8999512f/A: creating writer at 1731348980129Flushing mob file fba45d19b0840db7bf05326b8999512f/A: appending metadata at 1731348980133 (+4 ms)Flushing mob file fba45d19b0840db7bf05326b8999512f/A: closing flushed file at 1731348980133Flushing fba45d19b0840db7bf05326b8999512f/A: appending metadata at 1731348980540 (+407 ms)Flushing fba45d19b0840db7bf05326b8999512f/A: closing flushed file at 1731348980540Flushing fba45d19b0840db7bf05326b8999512f/B: creating writer at 1731348980947 (+407 ms)Flushing fba45d19b0840db7bf05326b8999512f/B: appending metadata at 1731348980949 (+2 ms)Flushing fba45d19b0840db7bf05326b8999512f/B: closing flushed file at 1731348980949Flushing fba45d19b0840db7bf05326b8999512f/C: creating writer at 1731348981356 (+407 ms)Flushing fba45d19b0840db7bf05326b8999512f/C: appending metadata at 1731348981357 (+1 ms)Flushing fba45d19b0840db7bf05326b8999512f/C: closing flushed file at 1731348981357Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15dd61df: reopening flushed file at 1731348981764 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@89d4391: reopening flushed file at 1731348981767 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e9bf5f2: reopening flushed file at 1731348981769 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fba45d19b0840db7bf05326b8999512f in 1644ms, sequenceid=965, compaction requested=true at 1731348981772 (+3 ms)Writing region close event to WAL at 1731348981824 (+52 ms)Running coprocessor post-close hooks at 1731348981827 (+3 ms)Closed at 1731348981827 2024-11-11T18:16:21,829 INFO [RS_CLOSE_REGION-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:21,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=fba45d19b0840db7bf05326b8999512f, regionState=CLOSED 2024-11-11T18:16:21,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 because future has completed 2024-11-11T18:16:21,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-11T18:16:21,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure fba45d19b0840db7bf05326b8999512f, server=90f34c5bee6a,42957,1731348800579 in 2.6680 sec 2024-11-11T18:16:21,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-11T18:16:21,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fba45d19b0840db7bf05326b8999512f, UNASSIGN in 2.6720 sec 2024-11-11T18:16:21,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-11T18:16:21,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6740 sec 2024-11-11T18:16:21,836 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731348981836"}]},"ts":"1731348981836"} 2024-11-11T18:16:21,837 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T18:16:21,838 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T18:16:21,839 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6820 sec 2024-11-11T18:16:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T18:16:23,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:23,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T18:16:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:23,326 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T18:16:23,327 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:23,328 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C, FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/recovered.edits] 2024-11-11T18:16:23,332 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1ba7e04b2ef748bdadbda935bc61ac54 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/1ba7e04b2ef748bdadbda935bc61ac54 2024-11-11T18:16:23,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/a41b06d011c5424fb532d1ae2341f23e to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/A/a41b06d011c5424fb532d1ae2341f23e 2024-11-11T18:16:23,335 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/604f1fc673cf4ea4a5e00f90ac069297 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/604f1fc673cf4ea4a5e00f90ac069297 2024-11-11T18:16:23,335 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ed75142259684ad48d33c5a4d84e1ad2 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/B/ed75142259684ad48d33c5a4d84e1ad2 2024-11-11T18:16:23,337 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/1b88a5e65c1041dfbba5afb405a0ef40 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/1b88a5e65c1041dfbba5afb405a0ef40 2024-11-11T18:16:23,338 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3cfe9f22eed043b08761107db85465f8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/3cfe9f22eed043b08761107db85465f8 2024-11-11T18:16:23,339 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/4f3a6129292b41879b725eb0c4c5e4fb to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/4f3a6129292b41879b725eb0c4c5e4fb 2024-11-11T18:16:23,340 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a453b3c9ef154796a2b69d8f7943b0ea to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/a453b3c9ef154796a2b69d8f7943b0ea 2024-11-11T18:16:23,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c7736b65d6fa42dabe63db42bad005f8 to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/C/c7736b65d6fa42dabe63db42bad005f8 2024-11-11T18:16:23,342 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/recovered.edits/968.seqid to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f/recovered.edits/968.seqid 2024-11-11T18:16:23,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/default/TestAcidGuarantees/fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,343 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T18:16:23,343 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T18:16:23,344 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T18:16:23,346 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a82f831cfa24101b1bd6bb525a0c861_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a82f831cfa24101b1bd6bb525a0c861_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,347 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111db003d2b15e463280188ad3ca61c11e_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111db003d2b15e463280188ad3ca61c11e_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,348 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111140f5d02ad8a54a3088f51741a1e02cfd_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111140f5d02ad8a54a3088f51741a1e02cfd_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,348 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111150365135374e46dabce6dd841ec8462e_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111150365135374e46dabce6dd841ec8462e_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,349 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111633ab1a44266482daa85d49ff549d4b8_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111633ab1a44266482daa85d49ff549d4b8_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,350 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173bb9852f9a240b5b66f13d98bdd5882_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111173bb9852f9a240b5b66f13d98bdd5882_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,351 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117dc2e6352d06446eb50378c680c6ed64_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117dc2e6352d06446eb50378c680c6ed64_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,352 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111180000fbcc1d444e1bd0b412c6d7cd33b_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111180000fbcc1d444e1bd0b412c6d7cd33b_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,352 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b9238cd514044f08a648a91cdcb2f8d_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118b9238cd514044f08a648a91cdcb2f8d_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,353 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118d410df912344e438462e8f12af37be2_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118d410df912344e438462e8f12af37be2_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,354 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111916d1aa2f281420bb585af3e4bbf4adc_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111916d1aa2f281420bb585af3e4bbf4adc_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,355 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a1b857716326412291df1cbfafa06677_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a1b857716326412291df1cbfafa06677_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,356 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b778b4f8a3a940b699a16a1e74f848e3_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b778b4f8a3a940b699a16a1e74f848e3_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,356 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be95aaceaae941928d56a4921b3838b0_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111be95aaceaae941928d56a4921b3838b0_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,357 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c07f47322dd74117930e47144ab3a5b5_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c07f47322dd74117930e47144ab3a5b5_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,358 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d77df6898af44dd0abf775839a1ce128_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d77df6898af44dd0abf775839a1ce128_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,359 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9eeae470f824eefa58a09197d04fd91_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9eeae470f824eefa58a09197d04fd91_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,360 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e44195864aa64f3f88206f26aeea73a2_fba45d19b0840db7bf05326b8999512f to hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e44195864aa64f3f88206f26aeea73a2_fba45d19b0840db7bf05326b8999512f 2024-11-11T18:16:23,360 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T18:16:23,362 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:23,364 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T18:16:23,366 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T18:16:23,366 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:23,367 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T18:16:23,367 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731348983367"}]},"ts":"9223372036854775807"} 2024-11-11T18:16:23,368 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T18:16:23,368 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fba45d19b0840db7bf05326b8999512f, NAME => 'TestAcidGuarantees,,1731348953944.fba45d19b0840db7bf05326b8999512f.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T18:16:23,368 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T18:16:23,368 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731348983368"}]},"ts":"9223372036854775807"} 2024-11-11T18:16:23,369 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T18:16:23,370 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T18:16:23,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-11T18:16:23,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41925 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T18:16:23,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T18:16:23,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T18:16:23,444 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=455 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=350 (was 346) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7388 (was 7511) 2024-11-11T18:16:23,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T18:16:23,444 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:16:23,444 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:16:23,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:23,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:23,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T18:16:23,445 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T18:16:23,445 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2051383393, stopped=false 2024-11-11T18:16:23,445 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=90f34c5bee6a,41925,1731348799760 2024-11-11T18:16:23,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T18:16:23,447 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T18:16:23,447 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:16:23,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:16:23,447 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T18:16:23,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T18:16:23,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:16:23,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:23,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T18:16:23,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T18:16:23,448 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '90f34c5bee6a,42957,1731348800579' ***** 2024-11-11T18:16:23,448 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T18:16:23,448 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T18:16:23,449 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(959): stopping server 90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;90f34c5bee6a:42957. 2024-11-11T18:16:23,449 DEBUG [RS:0;90f34c5bee6a:42957 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T18:16:23,449 DEBUG [RS:0;90f34c5bee6a:42957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T18:16:23,449 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T18:16:23,450 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T18:16:23,450 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T18:16:23,450 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T18:16:23,450 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T18:16:23,450 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T18:16:23,450 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T18:16:23,450 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T18:16:23,450 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:23,450 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T18:16:23,474 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/info/8c2ce11aaa3f4baa8461fa372b7be9c5 is 95, key is TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4./info:/1731348832351/DeleteFamily/seqid=0 2024-11-11T18:16:23,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742463_1639 (size=6514) 2024-11-11T18:16:23,519 INFO [regionserver/90f34c5bee6a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T18:16:23,651 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:23,851 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:23,878 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/info/8c2ce11aaa3f4baa8461fa372b7be9c5 2024-11-11T18:16:23,896 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/ns/3764a807fb624ea6a4790e178cd620af is 93, key is TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4./ns:/1731348832310/DeleteFamily/seqid=0 2024-11-11T18:16:23,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742464_1640 (size=5906) 2024-11-11T18:16:24,051 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:24,176 INFO [regionserver/90f34c5bee6a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T18:16:24,176 INFO [regionserver/90f34c5bee6a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T18:16:24,251 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:24,299 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/ns/3764a807fb624ea6a4790e178cd620af 2024-11-11T18:16:24,316 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/rep_barrier/4374d06bbb114fda9384e380a49c811f is 102, key is TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4./rep_barrier:/1731348832310/DeleteFamily/seqid=0 2024-11-11T18:16:24,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742465_1641 (size=6025) 2024-11-11T18:16:24,451 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:24,652 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:24,720 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/rep_barrier/4374d06bbb114fda9384e380a49c811f 2024-11-11T18:16:24,738 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/table/e8392b42ad9e43e6a7dc7cc1c0fd54d0 is 96, key is TestAcidGuarantees,,1731348804220.2330eb7332d026bb8d820d13c904c2d4./table:/1731348832310/DeleteFamily/seqid=0 2024-11-11T18:16:24,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742466_1642 (size=5936) 2024-11-11T18:16:24,852 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:25,052 DEBUG [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T18:16:25,141 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/table/e8392b42ad9e43e6a7dc7cc1c0fd54d0 2024-11-11T18:16:25,145 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/info/8c2ce11aaa3f4baa8461fa372b7be9c5 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/info/8c2ce11aaa3f4baa8461fa372b7be9c5 2024-11-11T18:16:25,148 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/info/8c2ce11aaa3f4baa8461fa372b7be9c5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T18:16:25,149 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/ns/3764a807fb624ea6a4790e178cd620af as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/ns/3764a807fb624ea6a4790e178cd620af 2024-11-11T18:16:25,151 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/ns/3764a807fb624ea6a4790e178cd620af, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T18:16:25,152 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/rep_barrier/4374d06bbb114fda9384e380a49c811f as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/rep_barrier/4374d06bbb114fda9384e380a49c811f 2024-11-11T18:16:25,154 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/rep_barrier/4374d06bbb114fda9384e380a49c811f, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T18:16:25,155 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/.tmp/table/e8392b42ad9e43e6a7dc7cc1c0fd54d0 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/table/e8392b42ad9e43e6a7dc7cc1c0fd54d0 2024-11-11T18:16:25,157 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/table/e8392b42ad9e43e6a7dc7cc1c0fd54d0, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T18:16:25,158 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false 2024-11-11T18:16:25,162 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T18:16:25,163 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T18:16:25,163 INFO [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T18:16:25,163 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731348983450Running coprocessor pre-close hooks at 1731348983450Disabling compacts and flushes for region at 1731348983450Disabling writes for close at 1731348983450Obtaining lock to block concurrent updates at 1731348983450Preparing flush snapshotting stores in 1588230740 at 1731348983450Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731348983451 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731348983451Flushing 1588230740/info: creating writer at 1731348983452 (+1 ms)Flushing 1588230740/info: appending metadata at 1731348983474 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731348983474Flushing 1588230740/ns: creating writer at 1731348983881 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731348983895 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731348983895Flushing 1588230740/rep_barrier: creating writer at 1731348984302 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731348984316 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731348984316Flushing 1588230740/table: creating writer at 1731348984723 (+407 ms)Flushing 1588230740/table: appending metadata at 1731348984737 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731348984737Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33f4e6f3: reopening flushed file at 1731348985144 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f72efa: reopening flushed file at 1731348985148 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b691012: reopening flushed file at 1731348985151 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8ad1f6: reopening flushed file at 1731348985155 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false at 1731348985158 (+3 ms)Writing region close event to WAL at 1731348985160 (+2 ms)Running coprocessor post-close hooks at 1731348985163 (+3 ms)Closed at 1731348985163 2024-11-11T18:16:25,163 DEBUG [RS_CLOSE_META-regionserver/90f34c5bee6a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T18:16:25,252 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(976): stopping server 90f34c5bee6a,42957,1731348800579; all regions closed. 2024-11-11T18:16:25,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741834_1010 (size=25344) 2024-11-11T18:16:25,258 DEBUG [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/oldWALs 2024-11-11T18:16:25,259 INFO [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 90f34c5bee6a%2C42957%2C1731348800579.meta:.meta(num 1731348803741) 2024-11-11T18:16:25,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741833_1009 (size=20693097) 2024-11-11T18:16:25,262 DEBUG [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/oldWALs 2024-11-11T18:16:25,262 INFO [RS:0;90f34c5bee6a:42957 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 90f34c5bee6a%2C42957%2C1731348800579:(num 1731348803248) 2024-11-11T18:16:25,262 DEBUG [RS:0;90f34c5bee6a:42957 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T18:16:25,262 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T18:16:25,262 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T18:16:25,262 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.ChoreService(370): Chore service for: regionserver/90f34c5bee6a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T18:16:25,262 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T18:16:25,262 INFO [regionserver/90f34c5bee6a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T18:16:25,263 INFO [RS:0;90f34c5bee6a:42957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42957 2024-11-11T18:16:25,267 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/90f34c5bee6a,42957,1731348800579 2024-11-11T18:16:25,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T18:16:25,267 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T18:16:25,271 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [90f34c5bee6a,42957,1731348800579] 2024-11-11T18:16:25,272 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/90f34c5bee6a,42957,1731348800579 already deleted, retry=false 2024-11-11T18:16:25,272 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 90f34c5bee6a,42957,1731348800579 expired; onlineServers=0 2024-11-11T18:16:25,273 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '90f34c5bee6a,41925,1731348799760' ***** 2024-11-11T18:16:25,273 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T18:16:25,273 INFO [M:0;90f34c5bee6a:41925 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T18:16:25,273 INFO [M:0;90f34c5bee6a:41925 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T18:16:25,273 DEBUG [M:0;90f34c5bee6a:41925 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T18:16:25,273 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T18:16:25,273 DEBUG [M:0;90f34c5bee6a:41925 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T18:16:25,273 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster-HFileCleaner.small.0-1731348802886 {}] cleaner.HFileCleaner(306): Exit Thread[master/90f34c5bee6a:0:becomeActiveMaster-HFileCleaner.small.0-1731348802886,5,FailOnTimeoutGroup] 2024-11-11T18:16:25,273 DEBUG [master/90f34c5bee6a:0:becomeActiveMaster-HFileCleaner.large.0-1731348802885 {}] cleaner.HFileCleaner(306): Exit Thread[master/90f34c5bee6a:0:becomeActiveMaster-HFileCleaner.large.0-1731348802885,5,FailOnTimeoutGroup] 2024-11-11T18:16:25,273 INFO [M:0;90f34c5bee6a:41925 {}] hbase.ChoreService(370): Chore service for: master/90f34c5bee6a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T18:16:25,273 INFO [M:0;90f34c5bee6a:41925 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T18:16:25,273 DEBUG [M:0;90f34c5bee6a:41925 {}] master.HMaster(1795): Stopping service threads 2024-11-11T18:16:25,273 INFO [M:0;90f34c5bee6a:41925 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T18:16:25,273 INFO [M:0;90f34c5bee6a:41925 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T18:16:25,274 ERROR [M:0;90f34c5bee6a:41925 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T18:16:25,274 INFO [M:0;90f34c5bee6a:41925 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T18:16:25,274 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T18:16:25,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T18:16:25,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T18:16:25,275 DEBUG [M:0;90f34c5bee6a:41925 {}] zookeeper.ZKUtil(347): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T18:16:25,275 WARN [M:0;90f34c5bee6a:41925 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T18:16:25,275 INFO [M:0;90f34c5bee6a:41925 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/.lastflushedseqids 2024-11-11T18:16:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742467_1643 (size=72) 2024-11-11T18:16:25,371 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T18:16:25,371 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42957-0x101083c082c0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T18:16:25,372 INFO [RS:0;90f34c5bee6a:42957 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T18:16:25,372 INFO [RS:0;90f34c5bee6a:42957 {}] regionserver.HRegionServer(1031): Exiting; stopping=90f34c5bee6a,42957,1731348800579; zookeeper connection closed. 2024-11-11T18:16:25,372 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7f6a0ec0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7f6a0ec0 2024-11-11T18:16:25,373 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T18:16:25,685 INFO [M:0;90f34c5bee6a:41925 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T18:16:25,685 INFO [M:0;90f34c5bee6a:41925 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T18:16:25,685 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T18:16:25,685 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:16:25,685 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:16:25,685 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T18:16:25,685 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:16:25,685 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=755.34 KB heapSize=923.98 KB 2024-11-11T18:16:25,701 DEBUG [M:0;90f34c5bee6a:41925 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/935a66e17dc844739d16ecf12115451e is 82, key is hbase:meta,,1/info:regioninfo/1731348803825/Put/seqid=0 2024-11-11T18:16:25,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742468_1644 (size=5672) 2024-11-11T18:16:26,104 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2074 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/935a66e17dc844739d16ecf12115451e 2024-11-11T18:16:26,124 DEBUG [M:0;90f34c5bee6a:41925 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f9ca8221d7644cba8450787d39ec736 is 2479, key is \x00\x00\x00\x00\x00\x00\x00)/proc:d/1731348835513/Put/seqid=0 2024-11-11T18:16:26,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742469_1645 (size=45687) 2024-11-11T18:16:26,528 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754.79 KB at sequenceid=2074 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f9ca8221d7644cba8450787d39ec736 2024-11-11T18:16:26,531 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f9ca8221d7644cba8450787d39ec736 2024-11-11T18:16:26,545 DEBUG [M:0;90f34c5bee6a:41925 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bcc06d7051542f1974642e110d18a6c is 69, key is 90f34c5bee6a,42957,1731348800579/rs:state/1731348803023/Put/seqid=0 2024-11-11T18:16:26,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073742470_1646 (size=5156) 2024-11-11T18:16:26,949 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2074 (bloomFilter=true), to=hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bcc06d7051542f1974642e110d18a6c 2024-11-11T18:16:26,952 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/935a66e17dc844739d16ecf12115451e as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/935a66e17dc844739d16ecf12115451e 2024-11-11T18:16:26,955 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/935a66e17dc844739d16ecf12115451e, entries=8, sequenceid=2074, filesize=5.5 K 2024-11-11T18:16:26,955 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f9ca8221d7644cba8450787d39ec736 as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f9ca8221d7644cba8450787d39ec736 2024-11-11T18:16:26,958 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f9ca8221d7644cba8450787d39ec736 2024-11-11T18:16:26,958 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f9ca8221d7644cba8450787d39ec736, entries=172, sequenceid=2074, filesize=44.6 K 2024-11-11T18:16:26,958 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6bcc06d7051542f1974642e110d18a6c as hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6bcc06d7051542f1974642e110d18a6c 2024-11-11T18:16:26,961 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39585/user/jenkins/test-data/1c7df5f9-ff93-c9f1-83e5-d3bf7973acf7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6bcc06d7051542f1974642e110d18a6c, entries=1, sequenceid=2074, filesize=5.0 K 2024-11-11T18:16:26,961 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(3140): Finished flush of dataSize ~755.34 KB/773469, heapSize ~923.68 KB/945848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2074, compaction requested=false 2024-11-11T18:16:26,963 INFO [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T18:16:26,963 DEBUG [M:0;90f34c5bee6a:41925 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731348985685Disabling compacts and flushes for region at 1731348985685Disabling writes for close at 1731348985685Obtaining lock to block concurrent updates at 1731348985685Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731348985685Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=773469, getHeapSize=946088, getOffHeapSize=0, getCellsCount=2271 at 1731348985685Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731348985686 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731348985686Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731348985700 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731348985700Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731348986108 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731348986123 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731348986123Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731348986531 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731348986545 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731348986545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be77d02: reopening flushed file at 1731348986951 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4a1e8d: reopening flushed file at 1731348986955 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f87ede4: reopening flushed file at 1731348986958 (+3 ms)Finished flush of dataSize ~755.34 KB/773469, heapSize ~923.68 KB/945848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2074, compaction requested=false at 1731348986961 (+3 ms)Writing region close event to WAL at 1731348986962 (+1 ms)Closed at 1731348986963 (+1 ms) 2024-11-11T18:16:26,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41191 is added to blk_1073741830_1006 (size=909486) 2024-11-11T18:16:26,965 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T18:16:26,965 INFO [M:0;90f34c5bee6a:41925 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T18:16:26,965 INFO [M:0;90f34c5bee6a:41925 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41925 2024-11-11T18:16:26,966 INFO [M:0;90f34c5bee6a:41925 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T18:16:27,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T18:16:27,068 INFO [M:0;90f34c5bee6a:41925 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T18:16:27,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41925-0x101083c082c0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T18:16:27,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T18:16:27,075 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T18:16:27,075 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T18:16:27,075 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T18:16:27,075 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/hadoop.log.dir/,STOPPED} 2024-11-11T18:16:27,077 WARN [BP-503899657-172.17.0.2-1731348796435 heartbeating to localhost/127.0.0.1:39585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T18:16:27,077 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T18:16:27,077 WARN [BP-503899657-172.17.0.2-1731348796435 heartbeating to localhost/127.0.0.1:39585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-503899657-172.17.0.2-1731348796435 (Datanode Uuid 89f9cc4a-99bd-4974-bb56-d9280a3d2ae7) service to localhost/127.0.0.1:39585 2024-11-11T18:16:27,077 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T18:16:27,079 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/data/data1/current/BP-503899657-172.17.0.2-1731348796435 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T18:16:27,080 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/cluster_e0217846-f742-e63a-c058-681c2da4328f/data/data2/current/BP-503899657-172.17.0.2-1731348796435 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T18:16:27,080 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T18:16:27,087 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T18:16:27,087 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T18:16:27,087 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T18:16:27,087 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T18:16:27,088 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/96612f6a-2822-dee5-f99a-2758317b002f/hadoop.log.dir/,STOPPED} 2024-11-11T18:16:27,109 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T18:16:27,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T18:16:27,339 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T18:16:27,341 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null